<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - loss.h</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font><font color='#0000FF'>#ifndef</font> DLIB_DNn_LOSS_H_
<font color='#0000FF'>#define</font> DLIB_DNn_LOSS_H_

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='loss_abstract.h.html'>loss_abstract.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='core.h.html'>core.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='utilities.h.html'>utilities.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../matrix.h.html'>../matrix.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../cuda/tensor_tools.h.html'>../cuda/tensor_tools.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../geometry.h.html'>../geometry.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../image_processing/box_overlap_testing.h.html'>../image_processing/box_overlap_testing.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../image_processing/full_object_detection.h.html'>../image_processing/full_object_detection.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../svm/ranking_tools.h.html'>../svm/ranking_tools.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>sstream<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>map<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>unordered_map<font color='#5555FF'>&gt;</font>

<font color='#0000FF'>namespace</font> dlib
<b>{</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_binary_hinge_'></a>loss_binary_hinge_</b> 
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> training_label_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> out_data[i];
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth, 
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>y <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>+</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> y <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>1</font>, "<font color='#CC0000'>y: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> y<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>-</font>y<font color='#5555FF'>*</font>out_data[i];
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>temp <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>temp;
                    g[i] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>scale<font color='#5555FF'>*</font>y;
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    g[i] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_binary_hinge_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_binary_hinge_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_binary_hinge_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_binary_hinge_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_binary_hinge_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_binary_hinge_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_binary_hinge</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_binary_hinge_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_binary_hinge/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_binary_hinge <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_binary_hinge_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_binary_log_'></a>loss_binary_log_</b> 
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> training_label_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> out_data[i];
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth, 
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>grad, output_tensor<font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>y: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> y<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> temp;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    temp <font color='#5555FF'>=</font> <font color='#BB00BB'>log1pexp</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>out_data[i]<font face='Lucida Console'>)</font>;
                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> y<font color='#5555FF'>*</font>scale<font color='#5555FF'>*</font>temp;
                    g[i] <font color='#5555FF'>=</font> y<font color='#5555FF'>*</font>scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>g[i]<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    temp <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>out_data[i]<font color='#5555FF'>-</font><font color='#BB00BB'>log1pexp</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>out_data[i]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font>y<font color='#5555FF'>*</font>scale<font color='#5555FF'>*</font>temp;
                    g[i] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>y<font color='#5555FF'>*</font>scale<font color='#5555FF'>*</font>g[i];
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_binary_log_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_binary_log_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_binary_log_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_binary_log_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_binary_log_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_binary_log_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_binary_log</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_binary_log_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_binary_log/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_binary_log <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_binary_log_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_multiclass_log_'></a>loss_multiclass_log_</b> 
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> training_label_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


            <font color='#009900'>// Note that output_tensor.k() should match the number of labels.
</font>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// The index of the largest output for this sample is the label.
</font>                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>index_of_max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rowm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>,i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth, 
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>grad, output_tensor<font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#009900'>// The network must produce a number of outputs that is equal to the number
</font>                <font color='#009900'>// of labels when using this type of loss.
</font>                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>y <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, "<font color='#CC0000'>y: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> y <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, output_tensor.k(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> i<font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>k;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>=</font><font color='#5555FF'>=</font> y<font face='Lucida Console'>)</font>
                    <b>{</b>
                        loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font color='#5555FF'>-</font><font color='#BB00BB'>safe_log</font><font face='Lucida Console'>(</font>g[idx]<font face='Lucida Console'>)</font>;
                        g[idx] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>g[idx]<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        g[idx] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>g[idx];
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_multiclass_log_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_multiclass_log_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_multiclass_log_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_multiclass_log_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_multiclass_log_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_multiclass_log</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_multiclass_log/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_multiclass_log <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_multiclass_log_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_multiclass_log_weighted_'></a>loss_multiclass_log_weighted_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> dlib::weighted_label<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> weighted_label;
        <font color='#0000FF'>typedef</font> weighted_label training_label_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


            <font color='#009900'>// Note that output_tensor.k() should match the number of labels.
</font>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// The index of the largest output for this sample is the label.
</font>                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>index_of_max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rowm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>,i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>grad, output_tensor<font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> wl <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> wl.label;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> weight <font color='#5555FF'>=</font> wl.weight;
                <font color='#009900'>// The network must produce a number of outputs that is equal to the number
</font>                <font color='#009900'>// of labels when using this type of loss.
</font>                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>y <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, "<font color='#CC0000'>y: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> y <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, output_tensor.k(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> i<font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>k;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>=</font><font color='#5555FF'>=</font> y<font face='Lucida Console'>)</font>
                    <b>{</b>
                        loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> weight<font color='#5555FF'>*</font>scale<font color='#5555FF'>*</font><font color='#5555FF'>-</font><font color='#BB00BB'>safe_log</font><font face='Lucida Console'>(</font>g[idx]<font face='Lucida Console'>)</font>;
                        g[idx] <font color='#5555FF'>=</font>weight<font color='#5555FF'>*</font>scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>g[idx]<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        g[idx] <font color='#5555FF'>=</font> weight<font color='#5555FF'>*</font>scale<font color='#5555FF'>*</font>g[idx];
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_weighted_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_multiclass_log_weighted_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_multiclass_log_weighted_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_multiclass_log_weighted_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_multiclass_log_weighted_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_multiclass_log_weighted_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_multiclass_log_weighted</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_weighted_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_multiclass_log_weighted/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_multiclass_log_weighted <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_multiclass_log_weighted_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_multimulticlass_log_'></a>loss_multimulticlass_log_</b> 
    <b>{</b>

    <font color='#0000FF'>public</font>:

        <b><a name='loss_multimulticlass_log_'></a>loss_multimulticlass_log_</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <b><a name='loss_multimulticlass_log_'></a>loss_multimulticlass_log_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::map<font color='#5555FF'>&lt;</font>std::string,std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> labels
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> l : labels<font face='Lucida Console'>)</font>
            <b>{</b>
                possible_labels[l.first] <font color='#5555FF'>=</font> std::make_shared<font color='#5555FF'>&lt;</font><font color='#BB00BB'>decltype</font><font face='Lucida Console'>(</font>l.second<font face='Lucida Console'>)</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>l.second<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>l.second.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>, "<font color='#CC0000'>Each classifier must have at least two possible labels.</font>"<font face='Lucida Console'>)</font>;

                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> l.second.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    label_idx_lookup[l.first][l.second[i]] <font color='#5555FF'>=</font> i;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>total_num_labels;
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='number_of_labels'></a>number_of_labels</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> total_num_labels; <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='number_of_classifiers'></a>number_of_classifiers</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> possible_labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        std::map<font color='#5555FF'>&lt;</font>std::string,std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <b><a name='get_labels'></a>get_labels</b> <font face='Lucida Console'>(</font> 
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b>
            std::map<font color='#5555FF'>&lt;</font>std::string,std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> info; 
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> i : possible_labels<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> label : <font color='#5555FF'>*</font>i.second<font face='Lucida Console'>)</font>
                    info[i.first].<font color='#BB00BB'>emplace_back</font><font face='Lucida Console'>(</font>label<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>return</font> info;
        <b>}</b>

        <font color='#0000FF'>class</font> <b><a name='classifier_output'></a>classifier_output</b>
        <b>{</b>

        <font color='#0000FF'>public</font>:
            <b><a name='classifier_output'></a>classifier_output</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

            <font color='#0000FF'><u>size_t</u></font> <b><a name='num_classes'></a>num_classes</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> class_probs.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

            <font color='#0000FF'><u>double</u></font> <b><a name='probability_of_class'></a>probability_of_class</b> <font face='Lucida Console'>(</font>
                <font color='#0000FF'><u>size_t</u></font> i
            <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
            <b>{</b> 
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>i <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>num_classes</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>return</font> <font color='#BB00BB'>class_probs</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>; 
            <b>}</b>

            <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> <b><a name='label'></a>label</b><font face='Lucida Console'>(</font>
                <font color='#0000FF'><u>size_t</u></font> i
            <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
            <b>{</b> 
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>i <font color='#5555FF'>&lt;</font> <font color='#BB00BB'>num_classes</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 
                <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>_labels<font face='Lucida Console'>)</font>[i]; 
            <b>}</b>

            <b><a name='operator'></a>operator</b> std::<b><a name='string'></a>string</b><font face='Lucida Console'>(</font>
            <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>num_classes</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>; 
                <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>_labels<font face='Lucida Console'>)</font>[<font color='#BB00BB'>index_of_max</font><font face='Lucida Console'>(</font>class_probs<font face='Lucida Console'>)</font>];
            <b>}</b>

            <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> classifier_output<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font>item.<font color='#BB00BB'>num_classes</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>; 
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>return</font> out;
            <b>}</b>

        <font color='#0000FF'>private</font>:

            <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='loss_multimulticlass_log_'></a>loss_multimulticlass_log_</b>;

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> EXP<font color='#5555FF'>&gt;</font>
            <b><a name='classifier_output'></a>classifier_output</b><font face='Lucida Console'>(</font>
                <font color='#0000FF'>const</font> matrix_exp<font color='#5555FF'>&lt;</font>EXP<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> class_probs,
                <font color='#0000FF'>const</font> std::shared_ptr<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> _labels
            <font face='Lucida Console'>)</font> : 
                class_probs<font face='Lucida Console'>(</font>class_probs<font face='Lucida Console'>)</font>, 
                _labels<font face='Lucida Console'>(</font>_labels<font face='Lucida Console'>)</font>
            <b>{</b>
            <b>}</b>

            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>1</font>,<font color='#979000'>0</font><font color='#5555FF'>&gt;</font> class_probs;
            std::shared_ptr<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> _labels;
        <b>}</b>;

        <font color='#0000FF'>typedef</font> std::map<font color='#5555FF'>&lt;</font>std::string,std::string<font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> std::map<font color='#5555FF'>&lt;</font>std::string,classifier_output<font color='#5555FF'>&gt;</font> output_label_type;


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter_begin
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>number_of_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You must give the loss_multimulticlass_log_'s constructor label data before you can use it!</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font><font color='#BB00BB'>number_of_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, "<font color='#CC0000'>The output tensor must have </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>number_of_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> channels.</font>"<font face='Lucida Console'>)</font>;


            <font color='#0000FF'><u>long</u></font> k_offset <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> l : possible_labels<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> iter <font color='#5555FF'>=</font> iter_begin;
                <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> classifier_name <font color='#5555FF'>=</font> l.first;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> labels <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>l.second<font face='Lucida Console'>)</font>; 
                scratch.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, scratch, <font color='#979000'>0</font>, output_tensor, k_offset, labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>scratch, scratch<font face='Lucida Console'>)</font>;

                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> scratch.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                    <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font>[classifier_name] <font color='#5555FF'>=</font> <font color='#BB00BB'>classifier_output</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rowm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>scratch<font face='Lucida Console'>)</font>,i<font face='Lucida Console'>)</font>, l.second<font face='Lucida Console'>)</font>;

                k_offset <font color='#5555FF'>+</font><font color='#5555FF'>=</font> labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth_begin, 
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>number_of_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You must give the loss_multimulticlass_log_'s constructor label data before you can use it!</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font><font color='#BB00BB'>number_of_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, "<font color='#CC0000'>The output tensor must have </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>number_of_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> channels.</font>"<font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>long</u></font> k_offset <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> l : label_idx_lookup<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> classifier_name <font color='#5555FF'>=</font> l.first;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> int_labels <font color='#5555FF'>=</font> l.second; 
                scratch.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, int_labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, scratch, <font color='#979000'>0</font>, output_tensor, k_offset, int_labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>scratch, scratch<font face='Lucida Console'>)</font>;


                <font color='#0000FF'>auto</font> truth <font color='#5555FF'>=</font> truth_begin;
                <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> scratch.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> scratch.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> int_labels.<font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>classifier_name<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>truth;

                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> scratch.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> i<font color='#5555FF'>*</font>scratch.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>k;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>=</font><font color='#5555FF'>=</font> y<font face='Lucida Console'>)</font>
                        <b>{</b>
                            loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font color='#5555FF'>-</font>std::<font color='#BB00BB'>log</font><font face='Lucida Console'>(</font>g[idx]<font face='Lucida Console'>)</font>;
                            g[idx] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>g[idx]<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            g[idx] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>g[idx];
                        <b>}</b>
                    <b>}</b>
                <b>}</b>

                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, grad, k_offset, scratch, <font color='#979000'>0</font>, int_labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                k_offset <font color='#5555FF'>+</font><font color='#5555FF'>=</font> int_labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>


        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multimulticlass_log_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_multimulticlass_log_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.<font color='#BB00BB'>get_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_multimulticlass_log_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_multimulticlass_log_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_multimulticlass_log_.</font>"<font face='Lucida Console'>)</font>;

            std::map<font color='#5555FF'>&lt;</font>std::string,std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> info; 
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>info, in<font face='Lucida Console'>)</font>;
            item <font color='#5555FF'>=</font> <font color='#BB00BB'>loss_multimulticlass_log_</font><font face='Lucida Console'>(</font>info<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_multimulticlass_log_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_multimulticlass_log, labels={</font>";
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> i <font color='#5555FF'>=</font> item.possible_labels.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; i <font color='#5555FF'>!</font><font color='#5555FF'>=</font> item.possible_labels.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> category <font color='#5555FF'>=</font> i<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font>first;
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> labels <font color='#5555FF'>=</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>i<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font>second<font face='Lucida Console'>)</font>;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> category <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>:(</font>";
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
                <b>{</b>
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> labels[j];
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>j<font color='#5555FF'>+</font><font color='#979000'>1</font> <font color='#5555FF'>&lt;</font> labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>,</font>";
                <b>}</b>

                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>+</font><font color='#5555FF'>+</font>i <font color='#5555FF'>!</font><font color='#5555FF'>=</font> item.possible_labels.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>";
            <b>}</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>}</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multimulticlass_log_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_multimulticlass_log&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n&lt;/loss_multimulticlass_log&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        std::map<font color='#5555FF'>&lt;</font>std::string,std::shared_ptr<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> possible_labels;
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> total_num_labels <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#009900'>// We make it true that: possible_labels[classifier][label_idx_lookup[classifier][label]] == label
</font>        std::map<font color='#5555FF'>&lt;</font>std::string, std::map<font color='#5555FF'>&lt;</font>std::string, <font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> label_idx_lookup;


        <font color='#009900'>// Scratch doesn't logically contribute to the state of this object.  It's just
</font>        <font color='#009900'>// temporary scratch space used by this class.  
</font>        <font color='#0000FF'>mutable</font> resizable_tensor scratch;


    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_multimulticlass_log <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_multimulticlass_log_, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>inline</font> <font color='#0000FF'><u>bool</u></font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> lhs, <font color='#0000FF'>const</font> loss_multimulticlass_log_::classifier_output<font color='#5555FF'>&amp;</font> rhs<font face='Lucida Console'>)</font>
    <b>{</b> <font color='#0000FF'>return</font> lhs <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>rhs<font face='Lucida Console'>)</font>; <b>}</b>
    <font color='#0000FF'>inline</font> <font color='#0000FF'><u>bool</u></font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multimulticlass_log_::classifier_output<font color='#5555FF'>&amp;</font> lhs, <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> rhs<font face='Lucida Console'>)</font>
    <b>{</b> <font color='#0000FF'>return</font> rhs <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>lhs<font face='Lucida Console'>)</font>; <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_multibinary_log_'></a>loss_multibinary_log_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>typedef</font> std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> output_label_type;

        <b><a name='loss_multibinary_log_'></a>loss_multibinary_log_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <b><a name='loss_multibinary_log_'></a>loss_multibinary_log_</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> gamma<font face='Lucida Console'>)</font> : gamma<font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>gamma <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// Note that output_tensor.k() should match the number of labels.
</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                output_label_type <font color='#BB00BB'>predictions</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    predictions[k] <font color='#5555FF'>=</font> out_data[i <font color='#5555FF'>*</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k];
                <b>}</b>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>predictions<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>grad, output_tensor<font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font> <font color='#5555FF'>/</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data  <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i, <font color='#5555FF'>+</font><font color='#5555FF'>+</font>truth<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> num_label_categories <font color='#5555FF'>=</font> truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> num_label_categories,
                    "<font color='#CC0000'>Number of label types should match the number of output channels. </font>"
                    "<font color='#CC0000'>output_tensor.k(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, num_label_categories: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_label_categories<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[k];
                    <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>y: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> y<font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> idx <font color='#5555FF'>=</font> i <font color='#5555FF'>*</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp <font color='#5555FF'>=</font> <font color='#BB00BB'>log1pexp</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>out_data[idx]<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> focus <font color='#5555FF'>=</font> std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> g[idx], gamma<font face='Lucida Console'>)</font>;
                        loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> y <font color='#5555FF'>*</font> scale <font color='#5555FF'>*</font> temp <font color='#5555FF'>*</font> focus;
                        g[idx] <font color='#5555FF'>=</font> y <font color='#5555FF'>*</font> scale <font color='#5555FF'>*</font> focus <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>g[idx] <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>gamma <font color='#5555FF'>*</font> temp <font color='#5555FF'>+</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>out_data[idx] <font color='#5555FF'>-</font> <font color='#BB00BB'>log1pexp</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>out_data[idx]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> focus <font color='#5555FF'>=</font> std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>g[idx], gamma<font face='Lucida Console'>)</font>;
                        loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font>y <font color='#5555FF'>*</font> scale <font color='#5555FF'>*</font> temp <font color='#5555FF'>*</font> focus;
                        g[idx] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>y <font color='#5555FF'>*</font> scale <font color='#5555FF'>*</font> focus <font color='#5555FF'>*</font> g[idx] <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>gamma <font color='#5555FF'>*</font> temp <font color='#5555FF'>+</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_gamma'></a>get_gamma</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> gamma; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multibinary_log_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_multibinary_log_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gamma, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_multibinary_log_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_multibinary_log_</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                item.gamma <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#0000FF'>return</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_multibinary_log_2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gamma, in<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_multibinary_log_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_multibinary_log_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_multibinary_log (gamma=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.gamma <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multibinary_log_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_multibinary_log gamma='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.gamma <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>double</u></font> gamma <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_multibinary_log <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_multibinary_log_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>enum</font> <font color='#0000FF'>class</font> <b><a name='use_image_pyramid'></a>use_image_pyramid</b> : uint8_t
    <b>{</b>
        no,
        yes
    <b>}</b>;

    <font color='#0000FF'>struct</font> <b><a name='mmod_options'></a>mmod_options</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>struct</font> <b><a name='detector_window_details'></a>detector_window_details</b>
        <b>{</b>
            <b><a name='detector_window_details'></a>detector_window_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>; 
            <b><a name='detector_window_details'></a>detector_window_details</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> w, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> h<font face='Lucida Console'>)</font> : width<font face='Lucida Console'>(</font>w<font face='Lucida Console'>)</font>, height<font face='Lucida Console'>(</font>h<font face='Lucida Console'>)</font> <b>{</b><b>}</b>
            <b><a name='detector_window_details'></a>detector_window_details</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> w, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> h, <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> l<font face='Lucida Console'>)</font> : width<font face='Lucida Console'>(</font>w<font face='Lucida Console'>)</font>, height<font face='Lucida Console'>(</font>h<font face='Lucida Console'>)</font>, label<font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> width <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> height <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            std::string label;

            <font color='#0000FF'>friend</font> <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> detector_window_details<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.width, out<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.height, out<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.label, out<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>friend</font> <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>detector_window_details<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font>
                    <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::mmod_options::detector_window_details</font>"<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.width, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.height, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font>
                    <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.label, in<font face='Lucida Console'>)</font>;
            <b>}</b>

        <b>}</b>;

        <b><a name='mmod_options'></a>mmod_options</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        std::vector<font color='#5555FF'>&lt;</font>detector_window_details<font color='#5555FF'>&gt;</font> detector_windows;
        <font color='#0000FF'><u>double</u></font> loss_per_false_alarm <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#0000FF'><u>double</u></font> loss_per_missed_target <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#0000FF'><u>double</u></font> truth_match_iou_threshold <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>;
        test_box_overlap overlaps_nms <font color='#5555FF'>=</font> <b><a name='test_box_overlap'></a>test_box_overlap</b><font face='Lucida Console'>(</font><font color='#979000'>0.4</font><font face='Lucida Console'>)</font>;
        test_box_overlap overlaps_ignore;
        <font color='#0000FF'><u>bool</u></font> use_bounding_box_regression <font color='#5555FF'>=</font> <font color='#979000'>false</font>; 
        <font color='#0000FF'><u>double</u></font> bbr_lambda <font color='#5555FF'>=</font> <font color='#979000'>100</font>; 
        <font color='#009900'>// This field is intentionally not serialized because I want people to really think hard
</font>        <font color='#009900'>// about ignoring the warnings that this suppresses.
</font>        <font color='#0000FF'><u>bool</u></font> be_quiet <font color='#5555FF'>=</font> <font color='#979000'>false</font>;

        use_image_pyramid assume_image_pyramid <font color='#5555FF'>=</font> use_image_pyramid::yes;

        <b><a name='mmod_options'></a>mmod_options</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> target_size,       <font color='#009900'>// We want the length of the longest dimension of the detector window to be this.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> min_target_size,   <font color='#009900'>// But we require that the smallest dimension of the detector window be at least this big.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> min_detector_window_overlap_iou <font color='#5555FF'>=</font> <font color='#979000'>0.75</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font> min_target_size <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> min_target_size <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> target_size<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0.5</font> <font color='#5555FF'>&lt;</font> min_detector_window_overlap_iou <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> min_detector_window_overlap_iou <font color='#5555FF'>&lt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// Figure out what detector windows we will need.
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> label : <font color='#BB00BB'>get_labels</font><font face='Lucida Console'>(</font>boxes<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> ratio : <font color='#BB00BB'>find_covering_aspect_ratios</font><font face='Lucida Console'>(</font>boxes, <font color='#BB00BB'>test_box_overlap</font><font face='Lucida Console'>(</font>min_detector_window_overlap_iou<font face='Lucida Console'>)</font>, label<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'><u>double</u></font> detector_width;
                    <font color='#0000FF'><u>double</u></font> detector_height;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>ratio <font color='#5555FF'>&lt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                    <b>{</b>
                        detector_height <font color='#5555FF'>=</font> target_size;
                        detector_width <font color='#5555FF'>=</font> ratio<font color='#5555FF'>*</font>target_size;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>detector_width <font color='#5555FF'>&lt;</font> min_target_size<font face='Lucida Console'>)</font>
                        <b>{</b>
                            detector_height <font color='#5555FF'>=</font> min_target_size<font color='#5555FF'>/</font>ratio;
                            detector_width <font color='#5555FF'>=</font> min_target_size;
                        <b>}</b>
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        detector_width <font color='#5555FF'>=</font> target_size;
                        detector_height <font color='#5555FF'>=</font> target_size<font color='#5555FF'>/</font>ratio;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>detector_height <font color='#5555FF'>&lt;</font> min_target_size<font face='Lucida Console'>)</font>
                        <b>{</b>
                            detector_width <font color='#5555FF'>=</font> min_target_size<font color='#5555FF'>*</font>ratio;
                            detector_height <font color='#5555FF'>=</font> min_target_size;
                        <b>}</b>
                    <b>}</b>

                    detector_window_details <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>std::<font color='#BB00BB'>round</font><font face='Lucida Console'>(</font>detector_width<font face='Lucida Console'>)</font>, <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>std::<font color='#BB00BB'>round</font><font face='Lucida Console'>(</font>detector_height<font face='Lucida Console'>)</font>, label<font face='Lucida Console'>)</font>;
                    detector_windows.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>p<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't call mmod_options's constructor with a set of boxes that is empty (or only contains ignored boxes).</font>"<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>set_overlap_nms</font><font face='Lucida Console'>(</font>boxes<font face='Lucida Console'>)</font>;
        <b>}</b>

        <b><a name='mmod_options'></a>mmod_options</b><font face='Lucida Console'>(</font>
            use_image_pyramid assume_image_pyramid,
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> min_detector_window_overlap_iou <font color='#5555FF'>=</font> <font color='#979000'>0.75</font>
        <font face='Lucida Console'>)</font>
            : assume_image_pyramid<font face='Lucida Console'>(</font>assume_image_pyramid<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>assume_image_pyramid <font color='#5555FF'>=</font><font color='#5555FF'>=</font> use_image_pyramid::no<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0.5</font> <font color='#5555FF'>&lt;</font> min_detector_window_overlap_iou <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> min_detector_window_overlap_iou <font color='#5555FF'>&lt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// Figure out what detector windows we will need.
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> label : <font color='#BB00BB'>get_labels</font><font face='Lucida Console'>(</font>boxes<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> rectangle : <font color='#BB00BB'>find_covering_rectangles</font><font face='Lucida Console'>(</font>boxes, <font color='#BB00BB'>test_box_overlap</font><font face='Lucida Console'>(</font>min_detector_window_overlap_iou<font face='Lucida Console'>)</font>, label<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    detector_windows.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#BB00BB'>detector_window_details</font><font face='Lucida Console'>(</font>rectangle.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rectangle.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, label<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't call mmod_options's constructor with a set of boxes that is empty (or only contains ignored boxes).</font>"<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>set_overlap_nms</font><font face='Lucida Console'>(</font>boxes<font face='Lucida Console'>)</font>;
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'><u>void</u></font> <b><a name='set_overlap_nms'></a>set_overlap_nms</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// Convert from mmod_rect to rectangle so we can call
</font>            <font color='#009900'>// find_tight_overlap_tester().
</font>            std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> temp;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> bi : boxes<font face='Lucida Console'>)</font>
            <b>{</b>
                std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font> rtemp;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> b : bi<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>b.ignore<font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;
                    rtemp.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>b.rect<font face='Lucida Console'>)</font>;
                <b>}</b>
                temp.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>rtemp<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            overlaps_nms <font color='#5555FF'>=</font> <font color='#BB00BB'>find_tight_overlap_tester</font><font face='Lucida Console'>(</font>temp<font face='Lucida Console'>)</font>;
            <font color='#009900'>// Relax the non-max-suppression a little so that it doesn't accidentally make
</font>            <font color='#009900'>// it impossible for the detector to output boxes matching the training data.
</font>            <font color='#009900'>// This could be a problem with the tightest possible nms test since there is
</font>            <font color='#009900'>// some small variability in how boxes get positioned between the training data
</font>            <font color='#009900'>// and the coordinate system used by the detector when it runs.  So relaxing it
</font>            <font color='#009900'>// here takes care of that.
</font>            <font color='#0000FF'>auto</font> iou_thresh             <font color='#5555FF'>=</font> <font color='#BB00BB'>advance_toward_1</font><font face='Lucida Console'>(</font>overlaps_nms.<font color='#BB00BB'>get_iou_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> percent_covered_thresh <font color='#5555FF'>=</font> <font color='#BB00BB'>advance_toward_1</font><font face='Lucida Console'>(</font>overlaps_nms.<font color='#BB00BB'>get_percent_covered_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            overlaps_nms <font color='#5555FF'>=</font> <font color='#BB00BB'>test_box_overlap</font><font face='Lucida Console'>(</font>iou_thresh, percent_covered_thresh<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>static</font> <font color='#0000FF'><u>double</u></font> <b><a name='advance_toward_1'></a>advance_toward_1</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>double</u></font> val
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>val <font color='#5555FF'>&lt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                val <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>val<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>0.1</font>;
            <font color='#0000FF'>return</font> val;
        <b>}</b>

        <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='count_overlaps'></a>count_overlaps</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> rects,
            <font color='#0000FF'>const</font> test_box_overlap<font color='#5555FF'>&amp;</font> overlaps,
            <font color='#0000FF'>const</font> rectangle<font color='#5555FF'>&amp;</font> ref_box
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>size_t</u></font> cnt <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b : rects<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>overlaps</font><font face='Lucida Console'>(</font>b, ref_box<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>cnt;
            <b>}</b>
            <font color='#0000FF'>return</font> cnt;
        <b>}</b>

        <font color='#0000FF'>static</font> std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font> <b><a name='find_rectangles_overlapping_all_others'></a>find_rectangles_overlapping_all_others</b> <font face='Lucida Console'>(</font>
            std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font> rects,
            <font color='#0000FF'>const</font> test_box_overlap<font color='#5555FF'>&amp;</font> overlaps
        <font face='Lucida Console'>)</font>
        <b>{</b>
            std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font> exemplars;
            dlib::rand rnd;

            <font color='#0000FF'>while</font><font face='Lucida Console'>(</font>rects.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Pick boxes at random and see if they overlap a lot of other boxes.  We will try
</font>                <font color='#009900'>// 500 different boxes each iteration and select whichever hits the most others to
</font>                <font color='#009900'>// add to our exemplar set.
</font>                rectangle best_ref_box;
                <font color='#0000FF'><u>size_t</u></font> best_cnt <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>500</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
                <b>{</b>
                    rectangle ref_box <font color='#5555FF'>=</font> rects[rnd.<font color='#BB00BB'>get_random_64bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>rects.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>];
                    <font color='#0000FF'><u>size_t</u></font> cnt <font color='#5555FF'>=</font> <font color='#BB00BB'>count_overlaps</font><font face='Lucida Console'>(</font>rects, overlaps, ref_box<font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>cnt <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> best_cnt<font face='Lucida Console'>)</font>
                    <b>{</b>
                        best_cnt <font color='#5555FF'>=</font> cnt;
                        best_ref_box <font color='#5555FF'>=</font> ref_box;
                    <b>}</b>
                <b>}</b>

                <font color='#009900'>// Now mark all the boxes the new ref box hit as hit.
</font>                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> rects.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>overlaps</font><font face='Lucida Console'>(</font>rects[i], best_ref_box<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#009900'>// remove box from rects so we don't hit it again later
</font>                        <font color='#BB00BB'>swap</font><font face='Lucida Console'>(</font>rects[i], rects.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                        rects.<font color='#BB00BB'>pop_back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                        <font color='#5555FF'>-</font><font color='#5555FF'>-</font>i;
                    <b>}</b>
                <b>}</b>

                exemplars.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>best_ref_box<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>return</font> exemplars;
        <b>}</b>

        <font color='#0000FF'>static</font> std::set<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font> <b><a name='get_labels'></a>get_labels</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> rects
        <font face='Lucida Console'>)</font>
        <b>{</b>
            std::set<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font> labels;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> rr : rects<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> r : rr<font face='Lucida Console'>)</font>
                    labels.<font color='#BB00BB'>insert</font><font face='Lucida Console'>(</font>r.label<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>return</font> labels;
        <b>}</b>

        <font color='#0000FF'>static</font> std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <b><a name='find_covering_aspect_ratios'></a>find_covering_aspect_ratios</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> rects,
            <font color='#0000FF'>const</font> test_box_overlap<font color='#5555FF'>&amp;</font> overlaps,
            <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> label
        <font face='Lucida Console'>)</font>
        <b>{</b>
            std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font> boxes;
            <font color='#009900'>// Make sure all the boxes have the same size and position, so that the only thing our
</font>            <font color='#009900'>// checks for overlap will care about is aspect ratio (i.e. scale and x,y position are
</font>            <font color='#009900'>// ignored).
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> bb : rects<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> b : bb<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>b.ignore <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> b.label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> label<font face='Lucida Console'>)</font>
                        boxes.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#BB00BB'>move_rect</font><font face='Lucida Console'>(</font><font color='#BB00BB'>set_rect_area</font><font face='Lucida Console'>(</font>b.rect,<font color='#979000'>400</font><font color='#5555FF'>*</font><font color='#979000'>400</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>point</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>

            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> ratios;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> r : <font color='#BB00BB'>find_rectangles_overlapping_all_others</font><font face='Lucida Console'>(</font>boxes, overlaps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                ratios.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>r.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font><font face='Lucida Console'>)</font>r.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> ratios;
        <b>}</b>

        <font color='#0000FF'>static</font> std::vector<font color='#5555FF'>&lt;</font>dlib::rectangle<font color='#5555FF'>&gt;</font> <b><a name='find_covering_rectangles'></a>find_covering_rectangles</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> rects,
            <font color='#0000FF'>const</font> test_box_overlap<font color='#5555FF'>&amp;</font> overlaps,
            <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> label
        <font face='Lucida Console'>)</font>
        <b>{</b>
            std::vector<font color='#5555FF'>&lt;</font>rectangle<font color='#5555FF'>&gt;</font> boxes;
            <font color='#009900'>// Make sure all the boxes have the same position, so that the we only check for
</font>            <font color='#009900'>// width and height.
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> bb : rects<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> b : bb<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>b.ignore <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> b.label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> label<font face='Lucida Console'>)</font>
                        boxes.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rectangle</font><font face='Lucida Console'>(</font>b.rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b.rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>

            <font color='#0000FF'>return</font> <font color='#BB00BB'>find_rectangles_overlapping_all_others</font><font face='Lucida Console'>(</font>boxes, overlaps<font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>;

    <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> mmod_options<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>4</font>;

        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.detector_windows, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.loss_per_false_alarm, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.loss_per_missed_target, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.truth_match_iou_threshold, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.overlaps_nms, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.overlaps_ignore, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>uint8_t<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>item.assume_image_pyramid<font face='Lucida Console'>)</font>, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.use_bounding_box_regression, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bbr_lambda, out<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>mmod_options<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> version <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> version <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>4</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::mmod_options</font>"<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> width;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> height;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>width, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>height, in<font face='Lucida Console'>)</font>;
            item.detector_windows <font color='#5555FF'>=</font> <b>{</b>mmod_options::<font color='#BB00BB'>detector_window_details</font><font face='Lucida Console'>(</font>width, height<font face='Lucida Console'>)</font><b>}</b>;
        <b>}</b>
        <font color='#0000FF'>else</font>
        <b>{</b>
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.detector_windows, in<font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.loss_per_false_alarm, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.loss_per_missed_target, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.truth_match_iou_threshold, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.overlaps_nms, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.overlaps_ignore, in<font face='Lucida Console'>)</font>;
        item.assume_image_pyramid <font color='#5555FF'>=</font> use_image_pyramid::yes;
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>3</font><font face='Lucida Console'>)</font>
        <b>{</b>
            uint8_t assume_image_pyramid <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>assume_image_pyramid, in<font face='Lucida Console'>)</font>;
            item.assume_image_pyramid <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>use_image_pyramid<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>assume_image_pyramid<font face='Lucida Console'>)</font>;
        <b>}</b>
        item.use_bounding_box_regression <font color='#5555FF'>=</font> <font color='#BB00BB'>mmod_options</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.use_bounding_box_regression; <font color='#009900'>// use default value since this wasn't provided
</font>        item.bbr_lambda <font color='#5555FF'>=</font> <font color='#BB00BB'>mmod_options</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.bbr_lambda; <font color='#009900'>// use default value since this wasn't provided
</font>        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>4</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.use_bounding_box_regression, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bbr_lambda, in<font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

    <font color='#0000FF'>inline</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>mmod_options::detector_window_details<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> detector_windows<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// write detector windows grouped by label
</font>        <font color='#009900'>// example output: aeroplane:74x30,131x30,70x45,54x70,198x30;bicycle:70x57,32x70,70x32,51x70,128x30,30x121;car:70x36,70x60,99x30,52x70,30x83,30x114,30x200
</font>
        std::map<font color='#5555FF'>&lt;</font>std::string, std::deque<font color='#5555FF'>&lt;</font>mmod_options::detector_window_details<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> detector_windows_by_label;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> detector_window : detector_windows<font face='Lucida Console'>)</font>
            detector_windows_by_label[detector_window.label].<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>detector_window<font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>size_t</u></font> label_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> i : detector_windows_by_label<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> label <font color='#5555FF'>=</font> i.first;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> detector_windows <font color='#5555FF'>=</font> i.second;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>label_count<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>;</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> label <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>:</font>";

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> detector_windows[j].width <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>x</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> detector_windows[j].height;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>j <font color='#5555FF'>+</font> <font color='#979000'>1</font> <font color='#5555FF'>&lt;</font> detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>,</font>";
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>return</font> out;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_mmod_'></a>loss_mmod_</b> 
    <b>{</b>
        <font color='#0000FF'>struct</font> <b><a name='intermediate_detection'></a>intermediate_detection</b>
        <b>{</b>
            <b><a name='intermediate_detection'></a>intermediate_detection</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>; 

            <b><a name='intermediate_detection'></a>intermediate_detection</b><font face='Lucida Console'>(</font>
                rectangle rect_
            <font face='Lucida Console'>)</font> : rect<font face='Lucida Console'>(</font>rect_<font face='Lucida Console'>)</font>, rect_bbr<font face='Lucida Console'>(</font>rect_<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <b><a name='intermediate_detection'></a>intermediate_detection</b><font face='Lucida Console'>(</font>
                rectangle rect_,
                <font color='#0000FF'><u>double</u></font> detection_confidence_,
                <font color='#0000FF'><u>size_t</u></font> tensor_offset_,
                <font color='#0000FF'><u>long</u></font> channel
            <font face='Lucida Console'>)</font> : rect<font face='Lucida Console'>(</font>rect_<font face='Lucida Console'>)</font>, detection_confidence<font face='Lucida Console'>(</font>detection_confidence_<font face='Lucida Console'>)</font>, tensor_offset<font face='Lucida Console'>(</font>tensor_offset_<font face='Lucida Console'>)</font>, tensor_channel<font face='Lucida Console'>(</font>channel<font face='Lucida Console'>)</font>, rect_bbr<font face='Lucida Console'>(</font>rect_<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <font color='#009900'>// rect is the rectangle you get without any bounding box regression.  So it's
</font>            <font color='#009900'>// the basic sliding window box (aka, the "anchor box").
</font>            rectangle rect;
            <font color='#0000FF'><u>double</u></font> detection_confidence <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>size_t</u></font> tensor_offset <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>long</u></font> tensor_channel <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#009900'>// rect_bbr = rect + bounding box regression.  So more accurate.  Or if bbr is off then
</font>            <font color='#009900'>// this is just rect.  The important thing about rect_bbr is that its the
</font>            <font color='#009900'>// rectangle we use for doing NMS.
</font>            drectangle rect_bbr; 
            <font color='#0000FF'><u>size_t</u></font> tensor_offset_dx <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>size_t</u></font> tensor_offset_dy <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>size_t</u></font> tensor_offset_dw <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>size_t</u></font> tensor_offset_dh <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'><u>bool</u></font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> intermediate_detection<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> detection_confidence <font color='#5555FF'>&lt;</font> item.detection_confidence; <b>}</b>
        <b>}</b>;

    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font> output_label_type;

        <b><a name='loss_mmod_'></a>loss_mmod_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <b><a name='loss_mmod_'></a>loss_mmod_</b><font face='Lucida Console'>(</font>mmod_options options_<font face='Lucida Console'>)</font> : options<font face='Lucida Console'>(</font>options_<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'>const</font> mmod_options<font color='#5555FF'>&amp;</font> <b><a name='get_options'></a>get_options</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> options; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter,
            <font color='#0000FF'><u>double</u></font> adjust_threshold <font color='#5555FF'>=</font> <font color='#979000'>0</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.use_bounding_box_regression<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>,  sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            std::vector<font color='#5555FF'>&lt;</font>intermediate_detection<font color='#5555FF'>&gt;</font> dets_accum;
            output_label_type final_dets;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>tensor_to_dets</font><font face='Lucida Console'>(</font>input_tensor, output_tensor, i, dets_accum, adjust_threshold, sub<font face='Lucida Console'>)</font>;

                <font color='#009900'>// Do non-max suppression
</font>                final_dets.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dets_accum.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>overlaps_any_box_nms</font><font face='Lucida Console'>(</font>final_dets, dets_accum[i].rect_bbr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;

                    final_dets.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mmod_rect</font><font face='Lucida Console'>(</font>dets_accum[i].rect_bbr,
                                                   dets_accum[i].detection_confidence,
                                                   options.detector_windows[dets_accum[i].tensor_channel].label<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>

                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>final_dets<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth, 
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.use_bounding_box_regression<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'><u>double</u></font> det_thresh_speed_adjust <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#009900'>// we will scale the loss so that it doesn't get really huge
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                g[i] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            std::vector<font color='#5555FF'>&lt;</font>intermediate_detection<font color='#5555FF'>&gt;</font> dets;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>tensor_to_dets</font><font face='Lucida Console'>(</font>input_tensor, output_tensor, i, dets, <font color='#5555FF'>-</font>options.loss_per_false_alarm <font color='#5555FF'>+</font> det_thresh_speed_adjust, sub<font face='Lucida Console'>)</font>;

                <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> max_num_dets <font color='#5555FF'>=</font> <font color='#979000'>50</font> <font color='#5555FF'>+</font> truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>5</font>;
                <font color='#009900'>// Prevent calls to tensor_to_dets() from running for a really long time
</font>                <font color='#009900'>// due to the production of an obscene number of detections.
</font>                <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> max_num_initial_dets <font color='#5555FF'>=</font> max_num_dets<font color='#5555FF'>*</font><font color='#979000'>100</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> max_num_initial_dets<font face='Lucida Console'>)</font>
                <b>{</b>
                    det_thresh_speed_adjust <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>det_thresh_speed_adjust,dets[max_num_initial_dets].detection_confidence <font color='#5555FF'>+</font> options.loss_per_false_alarm<font face='Lucida Console'>)</font>;
                <b>}</b>

                std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> truth_idxs;
                truth_idxs.<font color='#BB00BB'>reserve</font><font face='Lucida Console'>(</font>truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                std::unordered_map<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font>, rectangle<font color='#5555FF'>&gt;</font> idx_to_truth_rect;

                <font color='#009900'>// The loss will measure the number of incorrect detections.  A detection is
</font>                <font color='#009900'>// incorrect if it doesn't hit a truth rectangle or if it is a duplicate detection
</font>                <font color='#009900'>// on a truth rectangle.
</font>                loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>options.loss_per_missed_target;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> x : <font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>x.ignore<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>size_t</u></font> k;
                        point p;
                        <font color='#0000FF'>if</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_rect_to_feat_coord</font><font face='Lucida Console'>(</font>p, input_tensor, x, x.label, sub, k, options.assume_image_pyramid<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#009900'>// Ignore boxes that can't be detected by the CNN.
</font>                            loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> options.loss_per_missed_target;
                            truth_idxs.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>continue</font>;
                        <b>}</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>k<font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> i <font color='#5555FF'>=</font> idx_to_truth_rect.<font color='#BB00BB'>find</font><font face='Lucida Console'>(</font>idx<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i <font color='#5555FF'>!</font><font color='#5555FF'>=</font> idx_to_truth_rect.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>options.be_quiet<font face='Lucida Console'>)</font> 
                            <b>{</b>
                                <font color='#009900'>// Ignore duplicate truth box in feature coordinates.
</font>                                std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Warning, ignoring object.  We encountered a truth rectangle located at </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> x.rect;
                                std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, and we are ignoring it because it maps to the exact same feature coordinates </font>";
                                std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>as another truth rectangle located at </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> i<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font>second <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::endl;
                            <b>}</b>

                            loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> options.loss_per_missed_target;
                            truth_idxs.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>continue</font>;
                        <b>}</b>
                        loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> out_data[idx];
                        <font color='#009900'>// compute gradient
</font>                        g[idx] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>scale;
                        truth_idxs.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>idx<font face='Lucida Console'>)</font>;
                        idx_to_truth_rect[idx] <font color='#5555FF'>=</font> x.rect;
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        <font color='#009900'>// This box was ignored so shouldn't have been counted in the loss.
</font>                        loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> options.loss_per_missed_target;
                        truth_idxs.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>

                <font color='#009900'>// Measure the loss augmented score for the detections which hit a truth rect.
</font>                std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>truth_score_hits</font><font face='Lucida Console'>(</font>truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

                <font color='#009900'>// keep track of which truth boxes we have hit so far.
</font>                std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>hit_truth_table</font><font face='Lucida Console'>(</font>truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;

                std::vector<font color='#5555FF'>&lt;</font>intermediate_detection<font color='#5555FF'>&gt;</font> final_dets;
                <font color='#009900'>// The point of this loop is to fill out the truth_score_hits array. 
</font>                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> final_dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> max_num_dets; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>overlaps_any_box_nms</font><font face='Lucida Console'>(</font>final_dets, dets[i].rect_bbr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;

                    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> det_label <font color='#5555FF'>=</font> options.detector_windows[dets[i].tensor_channel].label;

                    <font color='#0000FF'>const</font> std::pair<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> hittruth <font color='#5555FF'>=</font> <font color='#BB00BB'>find_best_match</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth, hit_truth_table, dets[i].rect, det_label<font face='Lucida Console'>)</font>;

                    final_dets.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>dets[i].rect<font face='Lucida Console'>)</font>;

                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> truth_match <font color='#5555FF'>=</font> hittruth.first;
                    <font color='#009900'>// if hit truth rect
</font>                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>truth_match <font color='#5555FF'>&gt;</font> options.truth_match_iou_threshold<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#009900'>// if this is the first time we have seen a detect which hit (*truth)[hittruth.second]
</font>                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> score <font color='#5555FF'>=</font> dets[i].detection_confidence;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>hit_truth_table[hittruth.second] <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            hit_truth_table[hittruth.second] <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
                            truth_score_hits[hittruth.second] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> score;
                        <b>}</b>
                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            truth_score_hits[hittruth.second] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> score <font color='#5555FF'>+</font> options.loss_per_false_alarm;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>

                <font color='#009900'>// Check if any of the truth boxes are unobtainable because the NMS is
</font>                <font color='#009900'>// killing them.  If so, automatically set those unobtainable boxes to
</font>                <font color='#009900'>// ignore and print a warning message to the user.
</font>                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> hit_truth_table.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>hit_truth_table[i] <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font color='#5555FF'>!</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[i].ignore<font face='Lucida Console'>)</font> 
                    <b>{</b>
                        <font color='#009900'>// So we didn't hit this truth box.  Is that because there is
</font>                        <font color='#009900'>// another, different truth box, that overlaps it according to NMS?
</font>                        <font color='#0000FF'>const</font> std::pair<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> hittruth <font color='#5555FF'>=</font> <font color='#BB00BB'>find_best_match</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth, <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[i], i<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>hittruth.second <font color='#5555FF'>=</font><font color='#5555FF'>=</font> i <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[hittruth.second].ignore<font face='Lucida Console'>)</font>
                            <font color='#0000FF'>continue</font>;
                        rectangle best_matching_truth_box <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[hittruth.second];
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.<font color='#BB00BB'>overlaps_nms</font><font face='Lucida Console'>(</font>best_matching_truth_box, <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[i]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> idx <font color='#5555FF'>=</font> truth_idxs[i];
                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>idx <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#009900'>// We are ignoring this box so we shouldn't have counted it in the
</font>                                <font color='#009900'>// loss in the first place.  So we subtract out the loss values we
</font>                                <font color='#009900'>// added for it in the code above.
</font>                                loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> options.loss_per_missed_target<font color='#5555FF'>-</font>out_data[idx];
                                g[idx] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>options.be_quiet<font face='Lucida Console'>)</font> 
                                <b>{</b>
                                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Warning, ignoring object.  We encountered a truth rectangle located at </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[i].rect;
                                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> that is suppressed by non-max-suppression </font>";
                                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>because it is overlapped by another truth rectangle located at </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> best_matching_truth_box 
                                        <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> (IoU:</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>best_matching_truth_box,<font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[i]<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>, Percent covered:</font>" 
                                        <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>box_percent_covered</font><font face='Lucida Console'>(</font>best_matching_truth_box,<font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[i]<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>).</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::endl;
                                <b>}</b>
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>

                hit_truth_table.<font color='#BB00BB'>assign</font><font face='Lucida Console'>(</font>hit_truth_table.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
                final_dets.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

                <font color='#009900'>// Now figure out which detections jointly maximize the loss and detection score sum.  We
</font>                <font color='#009900'>// need to take into account the fact that allowing a true detection in the output, while 
</font>                <font color='#009900'>// initially reducing the loss, may allow us to increase the loss later with many duplicate
</font>                <font color='#009900'>// detections.
</font>                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> final_dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> max_num_dets; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>overlaps_any_box_nms</font><font face='Lucida Console'>(</font>final_dets, dets[i].rect_bbr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;

                    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> det_label <font color='#5555FF'>=</font> options.detector_windows[dets[i].tensor_channel].label;

                    <font color='#0000FF'>const</font> std::pair<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> hittruth <font color='#5555FF'>=</font> <font color='#BB00BB'>find_best_match</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth, hit_truth_table, dets[i].rect, det_label<font face='Lucida Console'>)</font>;

                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> truth_match <font color='#5555FF'>=</font> hittruth.first;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>truth_match <font color='#5555FF'>&gt;</font> options.truth_match_iou_threshold<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>truth_score_hits[hittruth.second] <font color='#5555FF'>&gt;</font> options.loss_per_missed_target<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>hit_truth_table[hittruth.second]<font face='Lucida Console'>)</font>
                            <b>{</b>
                                hit_truth_table[hittruth.second] <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
                                final_dets.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>dets[i]<font face='Lucida Console'>)</font>;
                                loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> options.loss_per_missed_target;

                                <font color='#009900'>// Now account for BBR loss and gradient if appropriate.
</font>                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.use_bounding_box_regression<font face='Lucida Console'>)</font>
                                <b>{</b>
                                    <font color='#0000FF'><u>double</u></font> dx <font color='#5555FF'>=</font> out_data[dets[i].tensor_offset_dx];
                                    <font color='#0000FF'><u>double</u></font> dy <font color='#5555FF'>=</font> out_data[dets[i].tensor_offset_dy];
                                    <font color='#0000FF'><u>double</u></font> dw <font color='#5555FF'>=</font> out_data[dets[i].tensor_offset_dw];
                                    <font color='#0000FF'><u>double</u></font> dh <font color='#5555FF'>=</font> out_data[dets[i].tensor_offset_dh];

                                    dpoint p <font color='#5555FF'>=</font> <font color='#BB00BB'>dcenter</font><font face='Lucida Console'>(</font>dets[i].rect<font face='Lucida Console'>)</font>;
                                    <font color='#0000FF'><u>double</u></font> w <font color='#5555FF'>=</font> dets[i].rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>;
                                    <font color='#0000FF'><u>double</u></font> h <font color='#5555FF'>=</font> dets[i].rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>;
                                    drectangle truth_box <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>[hittruth.second].rect;
                                    dpoint p_truth <font color='#5555FF'>=</font> <font color='#BB00BB'>dcenter</font><font face='Lucida Console'>(</font>truth_box<font face='Lucida Console'>)</font>; 

                                    <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>w <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                                    <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>h <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

                                    <font color='#0000FF'><u>double</u></font> target_dx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p_truth.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>w;
                                    <font color='#0000FF'><u>double</u></font> target_dy <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p_truth.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>h;
                                    <font color='#0000FF'><u>double</u></font> target_dw <font color='#5555FF'>=</font> std::<font color='#BB00BB'>log</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>truth_box.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>w<font face='Lucida Console'>)</font>;
                                    <font color='#0000FF'><u>double</u></font> target_dh <font color='#5555FF'>=</font> std::<font color='#BB00BB'>log</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>truth_box.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>h<font face='Lucida Console'>)</font>;


                                    <font color='#009900'>// compute smoothed L1 loss on BBR outputs.  This loss
</font>                                    <font color='#009900'>// is just the MSE loss when the loss is small and L1
</font>                                    <font color='#009900'>// when large.
</font>                                    dx <font color='#5555FF'>=</font> dx<font color='#5555FF'>-</font>target_dx;
                                    dy <font color='#5555FF'>=</font> dy<font color='#5555FF'>-</font>target_dy;
                                    dw <font color='#5555FF'>=</font> dw<font color='#5555FF'>-</font>target_dw;
                                    dh <font color='#5555FF'>=</font> dh<font color='#5555FF'>-</font>target_dh;

                                    <font color='#009900'>// use smoothed L1 
</font>                                    <font color='#0000FF'><u>double</u></font> ldx <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dx<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#979000'>1</font> ? <font color='#979000'>0.5</font><font color='#5555FF'>*</font>dx<font color='#5555FF'>*</font>dx : std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dx<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>0.5</font>;
                                    <font color='#0000FF'><u>double</u></font> ldy <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dy<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#979000'>1</font> ? <font color='#979000'>0.5</font><font color='#5555FF'>*</font>dy<font color='#5555FF'>*</font>dy : std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dy<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>0.5</font>;
                                    <font color='#0000FF'><u>double</u></font> ldw <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dw<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#979000'>1</font> ? <font color='#979000'>0.5</font><font color='#5555FF'>*</font>dw<font color='#5555FF'>*</font>dw : std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dw<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>0.5</font>;
                                    <font color='#0000FF'><u>double</u></font> ldh <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dh<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#979000'>1</font> ? <font color='#979000'>0.5</font><font color='#5555FF'>*</font>dh<font color='#5555FF'>*</font>dh : std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>dh<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>0.5</font>;

                                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> options.bbr_lambda<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>ldx <font color='#5555FF'>+</font> ldy <font color='#5555FF'>+</font> ldw <font color='#5555FF'>+</font> ldh<font face='Lucida Console'>)</font>;
      
                                    <font color='#009900'>// now compute the derivatives of the smoothed L1 loss
</font>                                    ldx <font color='#5555FF'>=</font> <font color='#BB00BB'>put_in_range</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>1</font>,<font color='#979000'>1</font>, dx<font face='Lucida Console'>)</font>;
                                    ldy <font color='#5555FF'>=</font> <font color='#BB00BB'>put_in_range</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>1</font>,<font color='#979000'>1</font>, dy<font face='Lucida Console'>)</font>;
                                    ldw <font color='#5555FF'>=</font> <font color='#BB00BB'>put_in_range</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>1</font>,<font color='#979000'>1</font>, dw<font face='Lucida Console'>)</font>;
                                    ldh <font color='#5555FF'>=</font> <font color='#BB00BB'>put_in_range</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>1</font>,<font color='#979000'>1</font>, dh<font face='Lucida Console'>)</font>;


                                    <font color='#009900'>// also smoothed L1 gradient goes to gradient output
</font>                                    g[dets[i].tensor_offset_dx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>options.bbr_lambda<font color='#5555FF'>*</font>ldx;
                                    g[dets[i].tensor_offset_dy] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>options.bbr_lambda<font color='#5555FF'>*</font>ldy;
                                    g[dets[i].tensor_offset_dw] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>options.bbr_lambda<font color='#5555FF'>*</font>ldw;
                                    g[dets[i].tensor_offset_dh] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>options.bbr_lambda<font color='#5555FF'>*</font>ldh;
                                <b>}</b>
                            <b>}</b>
                            <font color='#0000FF'>else</font>
                            <b>{</b>
                                final_dets.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>dets[i]<font face='Lucida Console'>)</font>;
                                loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> options.loss_per_false_alarm;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                    <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>overlaps_ignore_box</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth, dets[i].rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#009900'>// didn't hit anything
</font>                        final_dets.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>dets[i]<font face='Lucida Console'>)</font>;
                        loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> options.loss_per_false_alarm;
                    <b>}</b>
                <b>}</b>

                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> x : final_dets<font face='Lucida Console'>)</font>
                <b>{</b>
                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> out_data[x.tensor_offset];
                    g[x.tensor_offset] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale;
                <b>}</b>

                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>truth;
                g        <font color='#5555FF'>+</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                out_data <font color='#5555FF'>+</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b> <font color='#009900'>// END for (long i = 0; i &lt; output_tensor.num_samples(); ++i)
</font>

            <font color='#009900'>// Here we scale the loss so that it's roughly equal to the number of mistakes
</font>            <font color='#009900'>// in an image.  Note that this scaling is different than the scaling we
</font>            <font color='#009900'>// applied to the gradient but it doesn't matter since the loss value isn't
</font>            <font color='#009900'>// used to update parameters.  It's used only for display and to check if we
</font>            <font color='#009900'>// have converged.  So it doesn't matter that they are scaled differently and
</font>            <font color='#009900'>// this way the loss that is displayed is readily interpretable to the user.
</font>            <font color='#0000FF'>return</font> loss<font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>


        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mmod_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_mmod_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.options, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_mmod_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_mmod_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_mmod_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.options, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_mmod_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_mmod\t (</font>";

            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> opts <font color='#5555FF'>=</font> item.options;

            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>detector_windows:(</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.detector_windows <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";

            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, loss per FA:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.loss_per_false_alarm;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, loss per miss:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.loss_per_missed_target;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, truth match IOU thresh:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.truth_match_iou_threshold;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, use_bounding_box_regression:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.use_bounding_box_regression;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>opts.use_bounding_box_regression<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, bbr_lambda:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.bbr_lambda;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, overlaps_nms:(</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>opts.overlaps_nms.<font color='#BB00BB'>get_iou_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>,</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>opts.overlaps_nms.<font color='#BB00BB'>get_percent_covered_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, overlaps_ignore:(</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>opts.overlaps_ignore.<font color='#BB00BB'>get_iou_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>,</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>opts.overlaps_ignore.<font color='#BB00BB'>get_percent_covered_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>)</font>";

            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mmod_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// TODO, add options fields
</font>            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_mmod/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> net_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='tensor_to_dets'></a>tensor_to_dets</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor,
            <font color='#0000FF'><u>long</u></font> i,
            std::vector<font color='#5555FF'>&lt;</font>intermediate_detection<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> dets_accum,
            <font color='#0000FF'><u>double</u></font> adjust_threshold,
            <font color='#0000FF'>const</font> net_type<font color='#5555FF'>&amp;</font> net 
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>net.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>,net.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.use_bounding_box_regression<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>i;
            <font color='#009900'>// scan the final layer and output the positive scoring locations
</font>            dets_accum.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>double</u></font> score <font color='#5555FF'>=</font> out_data[<font face='Lucida Console'>(</font>k<font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c];
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>score <font color='#5555FF'>&gt;</font> adjust_threshold<font face='Lucida Console'>)</font>
                        <b>{</b>
                            dpoint p <font color='#5555FF'>=</font> <font color='#BB00BB'>output_tensor_to_input_tensor</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>point</font><font face='Lucida Console'>(</font>c,r<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                            drectangle rect <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_drect</font><font face='Lucida Console'>(</font>p, options.detector_windows[k].width, options.detector_windows[k].height<font face='Lucida Console'>)</font>;
                            rect <font color='#5555FF'>=</font> <font color='#BB00BB'>input_layer</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>tensor_space_to_image_space</font><font face='Lucida Console'>(</font>input_tensor,rect<font face='Lucida Console'>)</font>;

                            dets_accum.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font><font color='#BB00BB'>intermediate_detection</font><font face='Lucida Console'>(</font>rect, score, <font face='Lucida Console'>(</font>k<font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c, k<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.use_bounding_box_regression<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> offset <font color='#5555FF'>=</font> options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font color='#5555FF'>*</font><font color='#979000'>4</font>;
                                dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>offset<font color='#5555FF'>+</font><font color='#979000'>0</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
                                dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dy <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>offset<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
                                dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dw <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>offset<font color='#5555FF'>+</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
                                dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dh <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>offset<font color='#5555FF'>+</font><font color='#979000'>3</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;

                                <font color='#009900'>// apply BBR to dets_accum.back()
</font>                                <font color='#0000FF'><u>double</u></font> dx <font color='#5555FF'>=</font> out_data[dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dx];
                                <font color='#0000FF'><u>double</u></font> dy <font color='#5555FF'>=</font> out_data[dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dy];
                                <font color='#0000FF'><u>double</u></font> dw <font color='#5555FF'>=</font> out_data[dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dw];
                                <font color='#0000FF'><u>double</u></font> dh <font color='#5555FF'>=</font> out_data[dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.tensor_offset_dh];
                                dw <font color='#5555FF'>=</font> std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>dw<font face='Lucida Console'>)</font>;
                                dh <font color='#5555FF'>=</font> std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>dh<font face='Lucida Console'>)</font>;
                                <font color='#0000FF'><u>double</u></font> w <font color='#5555FF'>=</font> rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>;
                                <font color='#0000FF'><u>double</u></font> h <font color='#5555FF'>=</font> rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>;
                                rect <font color='#5555FF'>=</font> <font color='#BB00BB'>translate_rect</font><font face='Lucida Console'>(</font>rect, <font color='#BB00BB'>dpoint</font><font face='Lucida Console'>(</font>dx<font color='#5555FF'>*</font>w,dy<font color='#5555FF'>*</font>h<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                                rect <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_drect</font><font face='Lucida Console'>(</font>rect, w<font color='#5555FF'>*</font>dw<font color='#5555FF'>+</font><font color='#979000'>1</font>, h<font color='#5555FF'>*</font>dh<font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                                dets_accum.<font color='#BB00BB'>back</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.rect_bbr <font color='#5555FF'>=</font> rect;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            std::<font color='#BB00BB'>sort</font><font face='Lucida Console'>(</font>dets_accum.<font color='#BB00BB'>rbegin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dets_accum.<font color='#BB00BB'>rend</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>size_t</u></font> <b><a name='find_best_detection_window'></a>find_best_detection_window</b> <font face='Lucida Console'>(</font>
            rectangle rect,
            <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> label,
            use_image_pyramid assume_image_pyramid
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>assume_image_pyramid <font color='#5555FF'>=</font><font color='#5555FF'>=</font> use_image_pyramid::yes<font face='Lucida Console'>)</font>
            <b>{</b>
                rect <font color='#5555FF'>=</font> <font color='#BB00BB'>move_rect</font><font face='Lucida Console'>(</font><font color='#BB00BB'>set_rect_area</font><font face='Lucida Console'>(</font>rect, <font color='#979000'>400</font><font color='#5555FF'>*</font><font color='#979000'>400</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>point</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                rect <font color='#5555FF'>=</font> <font color='#BB00BB'>rectangle</font><font face='Lucida Console'>(</font>rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#009900'>// Figure out which detection window in options.detector_windows is most similar to rect
</font>            <font color='#009900'>// (in terms of aspect ratio, if assume_image_pyramid == use_image_pyramid::yes).
</font>            <font color='#0000FF'><u>size_t</u></font> best_i <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>double</u></font> best_ratio_diff <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>std::numeric_limits<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>infinity</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> options.detector_windows.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.detector_windows[i].label <font color='#5555FF'>!</font><font color='#5555FF'>=</font> label<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>continue</font>;

                rectangle det_window;
                
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.assume_image_pyramid <font color='#5555FF'>=</font><font color='#5555FF'>=</font> use_image_pyramid::yes<font face='Lucida Console'>)</font>
                <b>{</b>
                    det_window <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font><font color='#BB00BB'>point</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>, options.detector_windows[i].width, options.detector_windows[i].height<font face='Lucida Console'>)</font>;
                    det_window <font color='#5555FF'>=</font> <font color='#BB00BB'>move_rect</font><font face='Lucida Console'>(</font><font color='#BB00BB'>set_rect_area</font><font face='Lucida Console'>(</font>det_window, <font color='#979000'>400</font><font color='#5555FF'>*</font><font color='#979000'>400</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>point</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    det_window <font color='#5555FF'>=</font> <font color='#BB00BB'>rectangle</font><font face='Lucida Console'>(</font>options.detector_windows[i].width, options.detector_windows[i].height<font face='Lucida Console'>)</font>;
                <b>}</b>

                <font color='#0000FF'><u>double</u></font> iou <font color='#5555FF'>=</font> <font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>rect, det_window<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>iou <font color='#5555FF'>&gt;</font> best_ratio_diff<font face='Lucida Console'>)</font>
                <b>{</b>
                    best_ratio_diff <font color='#5555FF'>=</font> iou;
                    best_i <font color='#5555FF'>=</font> i;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> best_i;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> net_type<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>bool</u></font> <b><a name='image_rect_to_feat_coord'></a>image_rect_to_feat_coord</b> <font face='Lucida Console'>(</font>
            point<font color='#5555FF'>&amp;</font> tensor_p,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> rectangle<font color='#5555FF'>&amp;</font> rect,
            <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> label,
            <font color='#0000FF'>const</font> net_type<font color='#5555FF'>&amp;</font> net,
            <font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&amp;</font> det_idx,
            use_image_pyramid assume_image_pyramid
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b>
            <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>input_layer</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>image_contained_point</font><font face='Lucida Console'>(</font>input_tensor,<font color='#BB00BB'>center</font><font face='Lucida Console'>(</font>rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                std::ostringstream sout;
                sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Encountered a truth rectangle located at </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rect <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> that is outside the image.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                sout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>The center of each truth rectangle must be within the image.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>impossible_labeling_error</font><font face='Lucida Console'>(</font>sout.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            det_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>find_best_detection_window</font><font face='Lucida Console'>(</font>rect,label,assume_image_pyramid<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.assume_image_pyramid <font color='#5555FF'>=</font><font color='#5555FF'>=</font> use_image_pyramid::yes<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Compute the scale we need to be at to get from rect to our detection window.
</font>                <font color='#009900'>// Note that we compute the scale as the max of two numbers.  It doesn't
</font>                <font color='#009900'>// actually matter which one we pick, because if they are very different then
</font>                <font color='#009900'>// it means the box can't be matched by the sliding window.  But picking the
</font>                <font color='#009900'>// max causes the right error message to be selected in the logic below.
</font>                scale <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>options.detector_windows[det_idx].width<font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font><font face='Lucida Console'>)</font>rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, options.detector_windows[det_idx].height<font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font><font face='Lucida Console'>)</font>rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#009900'>// We don't want invariance to scale.
</font>                scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>;
            <b>}</b>

            <font color='#0000FF'>const</font> rectangle mapped_rect <font color='#5555FF'>=</font> <font color='#BB00BB'>input_layer</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>image_space_to_tensor_space</font><font face='Lucida Console'>(</font>input_tensor, std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font><font color='#979000'>1.0</font>,scale<font face='Lucida Console'>)</font>, rect<font face='Lucida Console'>)</font>;

            <font color='#009900'>// compute the detection window that we would use at this position.
</font>            tensor_p <font color='#5555FF'>=</font> <font color='#BB00BB'>center</font><font face='Lucida Console'>(</font>mapped_rect<font face='Lucida Console'>)</font>;
            rectangle det_window <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>tensor_p, options.detector_windows[det_idx].width,options.detector_windows[det_idx].height<font face='Lucida Console'>)</font>;
            det_window <font color='#5555FF'>=</font> <font color='#BB00BB'>input_layer</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>tensor_space_to_image_space</font><font face='Lucida Console'>(</font>input_tensor, det_window<font face='Lucida Console'>)</font>;

            <font color='#009900'>// make sure the rect can actually be represented by the image pyramid we are
</font>            <font color='#009900'>// using.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>rect, det_window<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> options.truth_match_iou_threshold<font face='Lucida Console'>)</font>
            <b>{</b>
                std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Warning, ignoring object.  We encountered a truth rectangle with a width and height of </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> and </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>.  </font>";
                std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>The image pyramid and sliding windows can't output a rectangle of this shape.  </font>";
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> detector_area <font color='#5555FF'>=</font> options.detector_windows[det_idx].width<font color='#5555FF'>*</font>options.detector_windows[det_idx].height;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mapped_rect.<font color='#BB00BB'>area</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>detector_area <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> options.truth_match_iou_threshold<font face='Lucida Console'>)</font>
                <b>{</b>
                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>This is because the rectangle is smaller than the best matching detection window, which has a width </font>";
                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>and height of </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> options.detector_windows[det_idx].width <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> and </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> options.detector_windows[det_idx].height <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::endl;
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>This is either because (1) the final layer's features have too large of a stride across the image, limiting the possible locations the sliding window can search </font>";
                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>or (2) because the rectangle's aspect ratio is too different from the best matching detection window, </font>";
                    std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>which has a width and height of </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> options.detector_windows[det_idx].width <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> and </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> options.detector_windows[det_idx].height <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::endl;
                <b>}</b>
                <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
            <b>}</b>

            <font color='#009900'>// now map through the CNN to the output layer.
</font>            tensor_p <font color='#5555FF'>=</font> <font color='#BB00BB'>input_tensor_to_output_tensor</font><font face='Lucida Console'>(</font>net,tensor_p<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> net.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font>tensor_p<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Warning, ignoring object.  We encountered a truth rectangle located at </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rect <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> that is too close to the edge </font>";
                std::cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>of the image to be captured by the CNN features.</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::endl;
                <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
            <b>}</b>

            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>


        <font color='#0000FF'><u>bool</u></font> <b><a name='overlaps_ignore_box'></a>overlaps_ignore_box</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes,
            <font color='#0000FF'>const</font> rectangle<font color='#5555FF'>&amp;</font> rect
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> b : boxes<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>b.ignore <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> options.<font color='#BB00BB'>overlaps_ignore</font><font face='Lucida Console'>(</font>b, rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
            <b>}</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>

        std::pair<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> <b><a name='find_best_match'></a>find_best_match</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes,
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> hit_truth_table,
            <font color='#0000FF'>const</font> rectangle<font color='#5555FF'>&amp;</font> rect,
            <font color='#0000FF'>const</font> std::string<font color='#5555FF'>&amp;</font> label
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'><u>double</u></font> match <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> best_idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> allow_duplicate_hit <font color='#5555FF'>=</font> <font color='#979000'>0</font>; allow_duplicate_hit <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> match <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>allow_duplicate_hit<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> boxes.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>boxes[i].ignore <font color='#5555FF'>|</font><font color='#5555FF'>|</font> boxes[i].label <font color='#5555FF'>!</font><font color='#5555FF'>=</font> label<font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>allow_duplicate_hit <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> hit_truth_table[i]<font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;

                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> new_match <font color='#5555FF'>=</font> <font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>rect, boxes[i]<font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>new_match <font color='#5555FF'>&gt;</font> match<font face='Lucida Console'>)</font>
                    <b>{</b>
                        match <font color='#5555FF'>=</font> new_match;
                        best_idx <font color='#5555FF'>=</font> i;
                    <b>}</b>
                <b>}</b>
            <b>}</b>

            <font color='#0000FF'>return</font> std::<font color='#BB00BB'>make_pair</font><font face='Lucida Console'>(</font>match,best_idx<font face='Lucida Console'>)</font>;
        <b>}</b>

        std::pair<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> <b><a name='find_best_match'></a>find_best_match</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes,
            <font color='#0000FF'>const</font> rectangle<font color='#5555FF'>&amp;</font> rect,
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> excluded_idx
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'><u>double</u></font> match <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> best_idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> boxes.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>boxes[i].ignore <font color='#5555FF'>|</font><font color='#5555FF'>|</font> excluded_idx <font color='#5555FF'>=</font><font color='#5555FF'>=</font> i<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>continue</font>;

                <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> new_match <font color='#5555FF'>=</font> <font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>rect, boxes[i]<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>new_match <font color='#5555FF'>&gt;</font> match<font face='Lucida Console'>)</font>
                <b>{</b>
                    match <font color='#5555FF'>=</font> new_match;
                    best_idx <font color='#5555FF'>=</font> i;
                <b>}</b>
            <b>}</b>

            <font color='#0000FF'>return</font> std::<font color='#BB00BB'>make_pair</font><font face='Lucida Console'>(</font>match,best_idx<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>inline</font> <font color='#0000FF'><u>bool</u></font> <b><a name='overlaps_any_box_nms'></a>overlaps_any_box_nms</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>T<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> rects,
            <font color='#0000FF'>const</font> rectangle<font color='#5555FF'>&amp;</font> rect
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> r : rects<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.<font color='#BB00BB'>overlaps_nms</font><font face='Lucida Console'>(</font>r.rect, rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
            <b>}</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>


        mmod_options options;

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_mmod <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_mmod_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_metric_'></a>loss_metric_</b> 
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> training_label_type;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> output_label_type;

        <b><a name='loss_metric_'></a>loss_metric_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <b><a name='loss_metric_'></a>loss_metric_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> margin_,
            <font color='#0000FF'><u>float</u></font> dist_thresh_
        <font face='Lucida Console'>)</font> : margin<font face='Lucida Console'>(</font>margin_<font face='Lucida Console'>)</font>, dist_thresh<font face='Lucida Console'>(</font>dist_thresh_<font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>margin_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dist_thresh_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> p <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#5555FF'>*</font>iter <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>p,output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;

                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter;
                p <font color='#5555FF'>+</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'><u>float</u></font> <b><a name='get_margin'></a>get_margin</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> margin; <b>}</b>
        <font color='#0000FF'><u>float</u></font> <b><a name='get_distance_threshold'></a>get_distance_threshold</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> dist_thresh; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth, 
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> 
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;



            temp.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            grad_mul.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>temp<font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, temp, <font color='#979000'>1</font>, output_tensor, <font color='#979000'>false</font>, output_tensor, <font color='#979000'>true</font><font face='Lucida Console'>)</font>;


            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> temp_threshs;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> d <font color='#5555FF'>=</font> temp.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>double</u></font> num_pos_samps <font color='#5555FF'>=</font> <font color='#979000'>0.0001</font>;
            <font color='#0000FF'><u>double</u></font> num_neg_samps <font color='#5555FF'>=</font> <font color='#979000'>0.0001</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> xx <font color='#5555FF'>=</font> d[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r];
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> x_label <font color='#5555FF'>=</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> r<font color='#5555FF'>+</font><font color='#979000'>1</font>; c <font color='#5555FF'>&lt;</font> temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> y_label <font color='#5555FF'>=</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> c<font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>x_label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> y_label<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_pos_samps;
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_neg_samps;

                        <font color='#009900'>// Figure out what distance threshold, when applied to the negative pairs,
</font>                        <font color='#009900'>// causes there to be an equal number of positive and negative pairs.
</font>                        <font color='#0000FF'>auto</font> yy <font color='#5555FF'>=</font> d[c<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c];
                        <font color='#0000FF'>auto</font> xy <font color='#5555FF'>=</font> d[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c];
                        <font color='#009900'>// compute the distance between x and y samples.
</font>                        <font color='#0000FF'>auto</font> d2 <font color='#5555FF'>=</font> xx <font color='#5555FF'>+</font> yy <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>xy;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>d2 <font color='#5555FF'>&lt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                            d2 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                        temp_threshs.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>d2<font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#009900'>// The whole objective function is multiplied by this to scale the loss
</font>            <font color='#009900'>// relative to the number of things in the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font color='#5555FF'>/</font>num_pos_samps;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num_pos_samps<font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font><font color='#979000'>1</font>, "<font color='#CC0000'>Make sure each mini-batch contains both positive pairs and negative pairs</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num_neg_samps<font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font><font color='#979000'>1</font>, "<font color='#CC0000'>Make sure each mini-batch contains both positive pairs and negative pairs</font>"<font face='Lucida Console'>)</font>;

            std::<font color='#BB00BB'>sort</font><font face='Lucida Console'>(</font>temp_threshs.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, temp_threshs.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> neg_thresh <font color='#5555FF'>=</font> std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>temp_threshs[std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font>num_pos_samps,num_neg_samps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>]<font face='Lucida Console'>)</font>;

            <font color='#009900'>// loop over all the pairs of training samples and compute the loss and
</font>            <font color='#009900'>// gradients.  Note that we only use the hardest negative pairs and that in
</font>            <font color='#009900'>// particular we pick the number of negative pairs equal to the number of
</font>            <font color='#009900'>// positive pairs so everything is balanced.
</font>            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> gm <font color='#5555FF'>=</font> grad_mul.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
            <b>{</b>
                gm[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> x_label <font color='#5555FF'>=</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> xx <font color='#5555FF'>=</font> d[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r];
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>r<font color='#5555FF'>=</font><font color='#5555FF'>=</font>c<font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;
                    <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> y_label <font color='#5555FF'>=</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> c<font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>auto</font> yy <font color='#5555FF'>=</font> d[c<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c];
                    <font color='#0000FF'>auto</font> xy <font color='#5555FF'>=</font> d[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c];

                    <font color='#009900'>// compute the distance between x and y samples.
</font>                    <font color='#0000FF'>auto</font> d2 <font color='#5555FF'>=</font> xx <font color='#5555FF'>+</font> yy <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>xy;
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>d2 <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                        d2 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                    <font color='#0000FF'>else</font> 
                        d2 <font color='#5555FF'>=</font> std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>d2<font face='Lucida Console'>)</font>;

                    <font color='#009900'>// It should be noted that the derivative of length(x-y) with respect
</font>                    <font color='#009900'>// to the x vector is the unit vector (x-y)/length(x-y).  If you stare
</font>                    <font color='#009900'>// at the code below long enough you will see that it's just an
</font>                    <font color='#009900'>// application of this formula.
</font>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>x_label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> y_label<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#009900'>// Things with the same label should have distances &lt; dist_thresh between
</font>                        <font color='#009900'>// them.  If not then we experience non-zero loss.
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>d2 <font color='#5555FF'>&lt;</font> dist_thresh<font color='#5555FF'>-</font>margin<font face='Lucida Console'>)</font>
                        <b>{</b>
                            gm[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                        <b>}</b>
                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>d2 <font color='#5555FF'>-</font> <font face='Lucida Console'>(</font>dist_thresh<font color='#5555FF'>-</font>margin<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                            gm[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>/</font>d2;
                            gm[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>scale<font color='#5555FF'>/</font>d2;
                        <b>}</b>
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        <font color='#009900'>// Things with different labels should have distances &gt; dist_thresh between
</font>                        <font color='#009900'>// them.  If not then we experience non-zero loss.
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>d2 <font color='#5555FF'>&gt;</font> dist_thresh<font color='#5555FF'>+</font>margin <font color='#5555FF'>|</font><font color='#5555FF'>|</font> d2 <font color='#5555FF'>&gt;</font> neg_thresh<font face='Lucida Console'>)</font>
                        <b>{</b>
                            gm[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                        <b>}</b>
                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>dist_thresh<font color='#5555FF'>+</font>margin<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> d2<font face='Lucida Console'>)</font>;
                            <font color='#009900'>// don't divide by zero (or a really small number)
</font>                            d2 <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>d2, <font color='#979000'>0.001f</font><font face='Lucida Console'>)</font>;
                            gm[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r] <font color='#5555FF'>-</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>/</font>d2;
                            gm[r<font color='#5555FF'>*</font>temp.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> scale<font color='#5555FF'>/</font>d2;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>


            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, grad, <font color='#979000'>1</font>, grad_mul, <font color='#979000'>false</font>, output_tensor, <font color='#979000'>false</font><font face='Lucida Console'>)</font>; 

            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_metric_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_metric_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.margin, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.dist_thresh, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_metric_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_metric_</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// These values used to be hard coded, so for this version of the metric
</font>                <font color='#009900'>// learning loss we just use these values.
</font>                item.margin <font color='#5555FF'>=</font> <font color='#979000'>0.1f</font>;
                item.dist_thresh <font color='#5555FF'>=</font> <font color='#979000'>0.75f</font>;
                <font color='#0000FF'>return</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_metric_2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.margin, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.dist_thresh, in<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_metric_.  Instead found </font>" <font color='#5555FF'>+</font> version<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_metric_<font color='#5555FF'>&amp;</font> item <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_metric (margin=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.margin<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>, distance_threshold=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.dist_thresh<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_metric_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_metric margin='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.margin<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>' distance_threshold='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.dist_thresh<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>float</u></font> margin <font color='#5555FF'>=</font> <font color='#979000'>0.04f</font>;
        <font color='#0000FF'><u>float</u></font> dist_thresh <font color='#5555FF'>=</font> <font color='#979000'>0.6f</font>;


        <font color='#009900'>// These variables are only here to avoid being reallocated over and over in
</font>        <font color='#009900'>// compute_loss_value_and_gradient()
</font>        <font color='#0000FF'>mutable</font> resizable_tensor temp, grad_mul;

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_metric <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_metric_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_ranking_'></a>loss_ranking_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> training_label_type; <font color='#009900'>// nominally +1/-1
</font>        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> output_label_type; <font color='#009900'>// ranking score
</font>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> out_data[i];
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;


            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> rel_scores;
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> nonrel_scores;
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> rel_idx, nonrel_idx;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    rel_scores.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>out_data[i]<font color='#5555FF'>-</font>y<font face='Lucida Console'>)</font>;
                    rel_idx.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
                <b>}</b>
                <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>y <font color='#5555FF'>&lt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    nonrel_scores.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>out_data[i]<font color='#5555FF'>-</font>y<font face='Lucida Console'>)</font>;
                    nonrel_idx.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font>;
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    g[i] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <b>}</b>
            <b>}</b>


            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> rel_counts;
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> nonrel_counts;
            <font color='#BB00BB'>count_ranking_inversions</font><font face='Lucida Console'>(</font>rel_scores, nonrel_scores, rel_counts, nonrel_counts<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> total_pairs <font color='#5555FF'>=</font> rel_scores.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>nonrel_scores.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>total_pairs <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't give a ranking mini-batch that contains only one class.  Both classes must be represented.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>total_pairs;


            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> rel_counts.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> rel_counts[k]<font color='#5555FF'>*</font>rel_scores[k];
                g[rel_idx[k]] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>1.0</font><font color='#5555FF'>*</font>rel_counts[k]<font color='#5555FF'>*</font>scale;
            <b>}</b>

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> nonrel_counts.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> nonrel_counts[k]<font color='#5555FF'>*</font>nonrel_scores[k];
                g[nonrel_idx[k]] <font color='#5555FF'>=</font> nonrel_counts[k]<font color='#5555FF'>*</font>scale;
            <b>}</b>

            <font color='#0000FF'>return</font> loss<font color='#5555FF'>*</font>scale;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_ranking_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_ranking_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_ranking_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_ranking_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_ranking_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_ranking_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_ranking</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_ranking_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_ranking/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_ranking <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_ranking_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_mean_squared_'></a>loss_mean_squared_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> training_label_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> out_data[i];
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp1 <font color='#5555FF'>=</font> y <font color='#5555FF'>-</font> out_data[i];
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp2 <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>temp1;
                loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> temp2<font color='#5555FF'>*</font>temp1;
                g[i] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>temp2;

            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_mean_squared_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_mean_squared_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_mean_squared_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_mean_squared_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_mean_squared_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_mean_squared</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_mean_squared/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_mean_squared <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_mean_squared_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_epsilon_insensitive_'></a>loss_epsilon_insensitive_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> training_label_type;
        <font color='#0000FF'>typedef</font> <font color='#0000FF'><u>float</u></font> output_label_type;

        <b><a name='loss_epsilon_insensitive_'></a>loss_epsilon_insensitive_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
        <b><a name='loss_epsilon_insensitive_'></a>loss_epsilon_insensitive_</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> eps<font face='Lucida Console'>)</font> : eps<font face='Lucida Console'>(</font>eps<font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>eps <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't set a negative error epsilon.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_epsilon'></a>get_epsilon</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> eps; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_epsilon'></a>set_epsilon</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> e<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>e <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>You can't set a negative error epsilon.</font>"<font face='Lucida Console'>)</font>;
            eps <font color='#5555FF'>=</font> e;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> out_data[i];
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> err <font color='#5555FF'>=</font> out_data[i]<font color='#5555FF'>-</font>y;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>err <font color='#5555FF'>&gt;</font> eps<font face='Lucida Console'>)</font>
                <b>{</b>
                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>err<font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font>;
                    g[i] <font color='#5555FF'>=</font> scale;
                <b>}</b>
                <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>err <font color='#5555FF'>&lt;</font> <font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font>
                <b>{</b>
                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>eps<font color='#5555FF'>-</font>err<font face='Lucida Console'>)</font>;
                    g[i] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>scale;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_epsilon_insensitive_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_epsilon_insensitive_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.eps, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_epsilon_insensitive_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_epsilon_insensitive_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_epsilon_insensitive_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.eps, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_epsilon_insensitive_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_epsilon_insensitive epsilon: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.eps;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_epsilon_insensitive_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_epsilon_insensitive_ epsilon='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.eps <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_epsilon_insensitive <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_epsilon_insensitive_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_mean_squared_multioutput_'></a>loss_mean_squared_multioutput_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_data, output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                out_data <font color='#5555FF'>+</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>; idx <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>idx<font face='Lucida Console'>)</font>
            <b>{</b>
                const_label_iterator truth_matrix_ptr <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> idx<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth_matrix_ptr<font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> k <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                             <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth_matrix_ptr<font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#009900'>// The loss we output is the average loss over the mini-batch.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> ytrue;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                ytrue <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> <font color='#BB00BB'>ytrue</font><font face='Lucida Console'>(</font>j, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp1 <font color='#5555FF'>=</font> y <font color='#5555FF'>-</font> <font color='#5555FF'>*</font>out_data<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp2 <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>temp1;
                    loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> temp2<font color='#5555FF'>*</font>temp1;
                    <font color='#5555FF'>*</font>g <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>temp2;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>g;
                <b>}</b>

            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_multioutput_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_mean_squared_multioutput_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_mean_squared_multioutput_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_mean_squared_multioutput_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_mean_squared_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_mean_squared_multioutput_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_mean_squared_multioutput</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_multioutput_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_mean_squared_multioutput/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_mean_squared_multioutput <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_mean_squared_multioutput_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_binary_log_per_pixel_'></a>loss_binary_log_per_pixel_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i, <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font> 
            <b>{</b>
                iter<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font> 
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font> 
                    <b>{</b>
                        iter<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>operator</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, i, <font color='#979000'>0</font>, r, c<font face='Lucida Console'>)</font>];
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>; idx <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>idx<font face='Lucida Console'>)</font>
            <b>{</b>
                const_label_iterator truth_matrix_ptr <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> idx<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                             truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                             "<font color='#CC0000'>truth size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>"
                             "<font color='#CC0000'>output size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'><u>double</u></font> loss;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#else</font>
            <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_binary_log_per_pixel_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_binary_log_per_pixel_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_binary_log_per_pixel_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_binary_log_per_pixel_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_binary_log_per_pixel_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_binary_log_per_pixel_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_binary_log_per_pixel</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_binary_log_per_pixel_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_binary_log_per_pixel/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        cuda::compute_loss_binary_log_per_pixel cuda_compute;
<font color='#0000FF'>#else</font>
        cpu::compute_loss_binary_log_per_pixel cpu_compute;
<font color='#0000FF'>#endif</font>
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_binary_log_per_pixel <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_binary_log_per_pixel_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_multiclass_log_per_pixel_'></a>loss_multiclass_log_per_pixel_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#009900'>// In semantic segmentation, if you don't know the ground-truth of some pixel,
</font>        <font color='#009900'>// set the label of that pixel to this value. When you do so, the pixel will be
</font>        <font color='#009900'>// ignored when computing gradients.
</font>        <font color='#0000FF'>static</font> <font color='#0000FF'>const</font> uint16_t label_to_ignore <font color='#5555FF'>=</font> std::numeric_limits<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font>::<b><a name='max'></a>max</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;


        <font color='#009900'>// In semantic segmentation, 65535 classes ought to be enough for anybody.
</font>        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>; <font color='#009900'>// Note that output_tensor.k() should match the number of labels.
</font>            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> std::numeric_limits<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// The index of the largest output for each element is the label.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> find_label <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> sample, <font color='#0000FF'><u>long</u></font> r, <font color='#0000FF'><u>long</u></font> c<font face='Lucida Console'>)</font> 
            <b>{</b>
                uint16_t label <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#0000FF'><u>float</u></font> max_value <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, sample, <font color='#979000'>0</font>, r, c<font face='Lucida Console'>)</font>];
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>1</font>; k <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font> 
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> value <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, sample, k, r, c<font face='Lucida Console'>)</font>];
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>value <font color='#5555FF'>&gt;</font> max_value<font face='Lucida Console'>)</font> 
                    <b>{</b>
                        label <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>k<font face='Lucida Console'>)</font>;
                        max_value <font color='#5555FF'>=</font> value;
                    <b>}</b>
                <b>}</b>
                <font color='#0000FF'>return</font> label;
            <b>}</b>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i, <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font> 
            <b>{</b>
                iter<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font> 
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font> 
                    <b>{</b>
                        <font color='#009900'>// The index of the largest output for this element is the label.
</font>                        iter<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>operator</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>find_label</font><font face='Lucida Console'>(</font>i, r, c<font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> std::numeric_limits<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>; idx <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>idx<font face='Lucida Console'>)</font>
            <b>{</b>
                const_label_iterator truth_matrix_ptr <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> idx<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                             truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                             "<font color='#CC0000'>truth size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>"
                             "<font color='#CC0000'>output size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>


            <font color='#0000FF'><u>double</u></font> loss;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#else</font>
            <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_per_pixel_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_multiclass_log_per_pixel_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_multiclass_log_per_pixel_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_multiclass_log_per_pixel_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_multiclass_log_per_pixel_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_multiclass_log_per_pixel_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_multiclass_log_per_pixel</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_per_pixel_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_multiclass_log_per_pixel/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        cuda::compute_loss_multiclass_log_per_pixel cuda_compute;
<font color='#0000FF'>#else</font>
        cpu::compute_loss_multiclass_log_per_pixel cpu_compute;
<font color='#0000FF'>#endif</font>
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_multiclass_log_per_pixel <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_multiclass_log_per_pixel_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_multiclass_log_per_pixel_weighted_'></a>loss_multiclass_log_per_pixel_weighted_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> dlib::weighted_label<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> weighted_label;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font>weighted_label<font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font>
        <b>{</b>
            loss_multiclass_log_per_pixel_::<font color='#BB00BB'>to_label</font><font face='Lucida Console'>(</font>input_tensor, sub, iter<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> std::numeric_limits<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>; idx <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>idx<font face='Lucida Console'>)</font>
            <b>{</b>
                const_label_iterator truth_matrix_ptr <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> idx<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                             truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                             "<font color='#CC0000'>truth size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>"
                             "<font color='#CC0000'>output size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'><u>double</u></font> loss;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#else</font>
            <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_per_pixel_weighted_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_multiclass_log_per_pixel_weighted_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_multiclass_log_per_pixel_weighted_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_multiclass_log_per_pixel_weighted_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_multiclass_log_per_pixel_weighted_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_multiclass_log_per_pixel_weighted_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_multiclass_log_per_pixel_weighted</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_multiclass_log_per_pixel_weighted_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_multiclass_log_per_pixel_weighted/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        cuda::compute_loss_multiclass_log_per_pixel_weighted cuda_compute;
<font color='#0000FF'>#else</font>
        cpu::compute_loss_multiclass_log_per_pixel_weighted cpu_compute;
<font color='#0000FF'>#endif</font>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_multiclass_log_per_pixel_weighted <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_multiclass_log_per_pixel_weighted_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_mean_squared_per_pixel_'></a>loss_mean_squared_per_pixel_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>output k = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i, <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
            <b>{</b>
                iter<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        iter<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>operator</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, i, <font color='#979000'>0</font>, r, c<font face='Lucida Console'>)</font>];
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> std::numeric_limits<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>; idx <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>idx<font face='Lucida Console'>)</font>
            <b>{</b>
                const_label_iterator truth_matrix_ptr <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> idx<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                    truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                    "<font color='#CC0000'>truth size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth_matrix_ptr<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>"
                    "<font color='#CC0000'>output size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#009900'>// The loss we output is the average loss over the mini-batch, and also over each element of the matrix output.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font> <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i, <font color='#5555FF'>+</font><font color='#5555FF'>+</font>truth<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>operator</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, i, <font color='#979000'>0</font>, r, c<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp1 <font color='#5555FF'>=</font> y <font color='#5555FF'>-</font> out_data[idx];
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> temp2 <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>temp1;
                        loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> temp2<font color='#5555FF'>*</font>temp1;
                        g[idx] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>temp2;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_per_pixel_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_mean_squared_per_pixel_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_mean_squared_per_pixel_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_mean_squared_per_pixel_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_mean_squared_per_pixel_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_mean_squared_per_pixel_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_mean_squared_per_pixel</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_per_pixel_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_mean_squared_per_pixel/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_mean_squared_per_pixel <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_mean_squared_per_pixel_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> _num_channels<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='loss_mean_squared_per_channel_and_pixel_'></a>loss_mean_squared_per_channel_and_pixel_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> std::array<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font>, _num_channels<font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> std::array<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font>, _num_channels<font color='#5555FF'>&gt;</font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> _num_channels, "<font color='#CC0000'>output k = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i, <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>iter<font face='Lucida Console'>)</font>[k].<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>iter<font face='Lucida Console'>)</font>[k].<font color='#0000FF'>operator</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, i, k, r, c<font face='Lucida Console'>)</font>];
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> _num_channels<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx <font color='#5555FF'>=</font> <font color='#979000'>0</font>; idx <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>idx<font face='Lucida Console'>)</font>
            <b>{</b>
                const_label_iterator truth_matrix_ptr <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>truth <font color='#5555FF'>+</font> idx<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth_matrix_ptr<font face='Lucida Console'>)</font>.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> _num_channels<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth_matrix_ptr<font face='Lucida Console'>)</font>[k].<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                        <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth_matrix_ptr<font face='Lucida Console'>)</font>[k].<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                        "<font color='#CC0000'>truth size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth_matrix_ptr<font face='Lucida Console'>)</font>[k].<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth_matrix_ptr<font face='Lucida Console'>)</font>[k].<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>"
                        "<font color='#CC0000'>output size = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> x </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'><u>double</u></font> loss;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#else</font>
            <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>truth, output_tensor, grad, loss<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
            <font color='#0000FF'>return</font> loss;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_per_channel_and_pixel_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_mean_squared_per_channel_and_pixel_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_mean_squared_per_channel_and_pixel_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_mean_squared_per_channel_and_pixel_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_mean_squared_per_channel_and_pixel_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_mean_squared_per_channel_and_pixel_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_mean_squared_per_channel_and_pixel</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_mean_squared_per_channel_and_pixel_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_mean_squared_per_channel_and_pixel/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        cuda::compute_loss_mean_squared_per_channel_and_pixel cuda_compute;
<font color='#0000FF'>#else</font>
        cpu::compute_loss_mean_squared_per_channel_and_pixel cpu_compute;
<font color='#0000FF'>#endif</font>
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> num_channels, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_mean_squared_per_channel_and_pixel <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_mean_squared_per_channel_and_pixel_<font color='#5555FF'>&lt;</font>num_channels<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_dot_'></a>loss_dot_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> output_label_type;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rowm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>,i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth, 
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> network_output_dims <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;


            <font color='#009900'>// The loss we output is the average loss over the mini-batch. 
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font color='#979000'>1.0</font><font color='#5555FF'>/</font>output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> network_output_dims, "<font color='#CC0000'>The network must output a vector with the same dimensionality as the training labels. </font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\ntruth-&gt;size():       </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth<font color='#5555FF'>-</font><font color='#5555FF'>&gt;</font><font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\nnetwork_output_dims: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> network_output_dims<font face='Lucida Console'>)</font>; 

                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> t <font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>truth<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;

                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> network_output_dims; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
                <b>{</b>
                    g[j] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>t[j]<font color='#5555FF'>*</font>scale;
                    loss <font color='#5555FF'>-</font><font color='#5555FF'>=</font> out_data[j]<font color='#5555FF'>*</font>t[j];
                <b>}</b>

                g <font color='#5555FF'>+</font><font color='#5555FF'>=</font> network_output_dims;
                out_data <font color='#5555FF'>+</font><font color='#5555FF'>=</font> network_output_dims;
            <b>}</b>
            <font color='#0000FF'>return</font> loss<font color='#5555FF'>*</font>scale;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_dot_<font color='#5555FF'>&amp;</font> , std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_dot_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_dot_<font color='#5555FF'>&amp;</font> , std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_dot_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_dot_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_dot_<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_dot</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_dot_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_dot/&gt;</font>";
        <b>}</b>

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_dot <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_dot_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>struct</font> <b><a name='yolo_options'></a>yolo_options</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>struct</font> <b><a name='anchor_box_details'></a>anchor_box_details</b>
        <b>{</b>
            <b><a name='anchor_box_details'></a>anchor_box_details</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;
            <b><a name='anchor_box_details'></a>anchor_box_details</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> w, <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> h<font face='Lucida Console'>)</font> : width<font face='Lucida Console'>(</font>w<font face='Lucida Console'>)</font>, height<font face='Lucida Console'>(</font>h<font face='Lucida Console'>)</font> <b>{</b><b>}</b>

            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> width <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> height <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'>friend</font> <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> anchor_box_details<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.width, out<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.height, out<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>friend</font> <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>anchor_box_details<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.width, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.height, in<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;

        <b><a name='yolo_options'></a>yolo_options</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='add_anchors'></a>add_anchors</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>anchor_box_details<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes<font face='Lucida Console'>)</font>
        <b>{</b>
            anchors[tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id] <font color='#5555FF'>=</font> boxes;
        <b>}</b>

        <font color='#009900'>// map between the stride and the anchor boxes
</font>        std::map<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font>, std::vector<font color='#5555FF'>&lt;</font>anchor_box_details<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> anchors;
        std::vector<font color='#5555FF'>&lt;</font>std::string<font color='#5555FF'>&gt;</font> labels;
        <font color='#0000FF'><u>double</u></font> iou_ignore_threshold <font color='#5555FF'>=</font> <font color='#979000'>0.7</font>;
        <font color='#0000FF'><u>double</u></font> iou_anchor_threshold <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>;
        test_box_overlap overlaps_nms <font color='#5555FF'>=</font> <b><a name='test_box_overlap'></a>test_box_overlap</b><font face='Lucida Console'>(</font><font color='#979000'>0.45</font>, <font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'><u>bool</u></font> classwise_nms <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <font color='#0000FF'><u>double</u></font> lambda_obj <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>;
        <font color='#0000FF'><u>double</u></font> lambda_box <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>;
        <font color='#0000FF'><u>double</u></font> lambda_cls <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>;

    <b>}</b>;

    <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> yolo_options<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>version, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.anchors, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.labels, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.iou_ignore_threshold, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.iou_anchor_threshold, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.classwise_nms, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.overlaps_nms, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.lambda_obj, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.lambda_box, out<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.lambda_cls, out<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>inline</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>yolo_options<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>int</u></font> version <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
            <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::yolo_options.</font>"<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.anchors, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.labels, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.iou_ignore_threshold, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.iou_anchor_threshold, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.classwise_nms, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.overlaps_nms, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.lambda_obj, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.lambda_box, in<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.lambda_cls, in<font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'>inline</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> std::map<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font>, std::vector<font color='#5555FF'>&lt;</font>yolo_options::anchor_box_details<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> anchors<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// write anchor boxes grouped by tag id
</font>        <font color='#0000FF'><u>size_t</u></font> tag_count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> i : anchors<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> tag_id <font color='#5555FF'>=</font> i.first;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> details <font color='#5555FF'>=</font> i.second;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>tag_count<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>;</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>tag</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> tag_id <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>:</font>";
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> a <font color='#5555FF'>=</font> <font color='#979000'>0</font>; a <font color='#5555FF'>&lt;</font> details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>a<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> details[a].width <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>x</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> details[a].height;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>a <font color='#5555FF'>+</font> <font color='#979000'>1</font> <font color='#5555FF'>&lt;</font> details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>,</font>";
            <b>}</b>
        <b>}</b>
        <font color='#0000FF'>return</font> out;
    <b>}</b>

    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b>, <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font>... <b><a name='TAG_TYPES'></a>TAG_TYPES</b><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='yolo_helper_impl'></a>yolo_helper_impl</b>
        <b>{</b>
            constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>return</font> <font color='#979000'>1</font> <font color='#5555FF'>+</font> yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>tag</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font> ? "<font color='#CC0000'>,</font>" : "<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
                yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='tensor_to_dets'></a>tensor_to_dets</b> <font face='Lucida Console'>(</font>
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
                <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub,
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n,
                <font color='#0000FF'>const</font> yolo_options<font color='#5555FF'>&amp;</font> options,
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> adjust_threshold,
                std::vector<font color='#5555FF'>&lt;</font>yolo_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> dets
            <font face='Lucida Console'>)</font>
            <b>{</b>
                yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tensor_to_dets</font><font face='Lucida Console'>(</font>input_tensor, sub, n, options, adjust_threshold, dets<font face='Lucida Console'>)</font>;
                yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tensor_to_dets</font><font face='Lucida Console'>(</font>input_tensor, sub, n, options, adjust_threshold, dets<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> const_label_iterator,
                <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='tensor_to_loss'></a>tensor_to_loss</b> <font face='Lucida Console'>(</font>
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
                const_label_iterator truth,
                SUBNET<font color='#5555FF'>&amp;</font> sub,
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n,
                <font color='#0000FF'>const</font> yolo_options<font color='#5555FF'>&amp;</font> options,
                <font color='#0000FF'><u>double</u></font><font color='#5555FF'>&amp;</font> loss
            <font face='Lucida Console'>)</font>
            <b>{</b>
                yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tensor_to_loss</font><font face='Lucida Console'>(</font>input_tensor, truth, sub, n, options, loss<font face='Lucida Console'>)</font>;
                yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tensor_to_loss</font><font face='Lucida Console'>(</font>input_tensor, truth, sub, n, options, loss<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='yolo_helper_impl'></a>yolo_helper_impl</b><font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>
        <b>{</b>
            constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#979000'>1</font>; <b>}</b>

            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font> <b>{</b> out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>tag</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id; <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='tensor_to_dets'></a>tensor_to_dets</b> <font face='Lucida Console'>(</font>
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
                <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub,
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n,
                <font color='#0000FF'>const</font> yolo_options<font color='#5555FF'>&amp;</font> options,
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> adjust_threshold,
                std::vector<font color='#5555FF'>&lt;</font>yolo_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> dets
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> anchors <font color='#5555FF'>=</font> options.anchors.<font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> anchors.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>options.labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> stride_x <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> stride_y <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_feats <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> anchors.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_classes <font color='#5555FF'>=</font> num_feats <font color='#5555FF'>-</font> <font color='#979000'>5</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> a <font color='#5555FF'>=</font> <font color='#979000'>0</font>; a <font color='#5555FF'>&lt;</font> anchors.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>a<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> a <font color='#5555FF'>*</font> num_feats;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> obj <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>4</font>, r, c<font face='Lucida Console'>)</font>];
                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>obj <font color='#5555FF'>&gt;</font> adjust_threshold<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> x <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>0</font>, r, c<font face='Lucida Console'>)</font>] <font color='#5555FF'>*</font> <font color='#979000'>2.0</font> <font color='#5555FF'>-</font> <font color='#979000'>0.5</font>;
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> y <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>1</font>, r, c<font face='Lucida Console'>)</font>] <font color='#5555FF'>*</font> <font color='#979000'>2.0</font> <font color='#5555FF'>-</font> <font color='#979000'>0.5</font>;
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> w <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>2</font>, r, c<font face='Lucida Console'>)</font>];
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> h <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font>];
                                yolo_rect <font color='#BB00BB'>det</font><font face='Lucida Console'>(</font><font color='#BB00BB'>centered_drect</font><font face='Lucida Console'>(</font><font color='#BB00BB'>dpoint</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>x <font color='#5555FF'>+</font> c<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> stride_x, <font face='Lucida Console'>(</font>y <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> stride_y<font face='Lucida Console'>)</font>,
                                                             w <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> w<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> anchors[a].width,
                                                             h <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> h<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> anchors[a].height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                                <b>{</b>
                                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> conf <font color='#5555FF'>=</font> obj <font color='#5555FF'>*</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>5</font> <font color='#5555FF'>+</font> i, r, c<font face='Lucida Console'>)</font>];
                                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>conf <font color='#5555FF'>&gt;</font> adjust_threshold<font face='Lucida Console'>)</font>
                                        det.labels.<font color='#BB00BB'>emplace_back</font><font face='Lucida Console'>(</font>conf, options.labels[i]<font face='Lucida Console'>)</font>;
                                <b>}</b>
                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>det.labels.<font color='#BB00BB'>empty</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                                <b>{</b>
                                    std::<font color='#BB00BB'>sort</font><font face='Lucida Console'>(</font>det.labels.<font color='#BB00BB'>rbegin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, det.labels.<font color='#BB00BB'>rend</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                                    det.detection_confidence <font color='#5555FF'>=</font> det.labels[<font color='#979000'>0</font>].first;
                                    det.label <font color='#5555FF'>=</font> det.labels[<font color='#979000'>0</font>].second;
                                    dets.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>det<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                                <b>}</b>
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>

            <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
                <font color='#0000FF'>typename</font> const_label_iterator,
                <font color='#0000FF'>typename</font> SUBNET
            <font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='tensor_to_loss'></a>tensor_to_loss</b> <font face='Lucida Console'>(</font>
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
                const_label_iterator truth,
                SUBNET<font color='#5555FF'>&amp;</font> sub,
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n,
                <font color='#0000FF'>const</font> yolo_options<font color='#5555FF'>&amp;</font> options,
                <font color='#0000FF'><u>double</u></font><font color='#5555FF'>&amp;</font> loss
            <font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> anchors <font color='#5555FF'>=</font> options.anchors.<font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> anchors.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>options.labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> stride_x <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> stride_y <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_feats <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> anchors.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_classes <font color='#5555FF'>=</font> num_feats <font color='#5555FF'>-</font> <font color='#979000'>5</font>;
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>const</font> rectangle <font color='#BB00BB'>input_rect</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, input_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

                <font color='#009900'>// Compute the objectness loss for all grid cells
</font>                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> a <font color='#5555FF'>=</font> <font color='#979000'>0</font>; a <font color='#5555FF'>&lt;</font> anchors.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>a<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> a <font color='#5555FF'>*</font> num_feats;
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> x <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>0</font>, r, c<font face='Lucida Console'>)</font>] <font color='#5555FF'>*</font> <font color='#979000'>2.0</font> <font color='#5555FF'>-</font> <font color='#979000'>0.5</font>;
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> y <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>1</font>, r, c<font face='Lucida Console'>)</font>] <font color='#5555FF'>*</font> <font color='#979000'>2.0</font> <font color='#5555FF'>-</font> <font color='#979000'>0.5</font>;
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> w <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>2</font>, r, c<font face='Lucida Console'>)</font>];
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> h <font color='#5555FF'>=</font> out_data[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font>];

                            <font color='#009900'>// The prediction at r, c for anchor a
</font>                            <font color='#0000FF'>const</font> yolo_rect <font color='#BB00BB'>pred</font><font face='Lucida Console'>(</font><font color='#BB00BB'>centered_drect</font><font face='Lucida Console'>(</font><font color='#BB00BB'>dpoint</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>x <font color='#5555FF'>+</font> c<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> stride_x, <font face='Lucida Console'>(</font>y <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> stride_y<font face='Lucida Console'>)</font>,
                                                                w <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> w<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> anchors[a].width,
                                                                h <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> h<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> anchors[a].height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                            <font color='#009900'>// Find the best IoU for all ground truth boxes
</font>                            <font color='#0000FF'><u>double</u></font> best_iou <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> yolo_rect<font color='#5555FF'>&amp;</font> truth_box : <font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>truth_box.ignore <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#5555FF'>!</font>input_rect.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font><font color='#BB00BB'>center</font><font face='Lucida Console'>(</font>truth_box.rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                                    <font color='#0000FF'>continue</font>;
                                best_iou <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>best_iou, <font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>truth_box.rect, pred.rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                            <b>}</b>

                            <font color='#009900'>// Incur loss for the boxes that are below a certain IoU threshold with any truth box
</font>                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> o_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>4</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>best_iou <font color='#5555FF'>&lt;</font> options.iou_ignore_threshold<font face='Lucida Console'>)</font>
                                g[o_idx] <font color='#5555FF'>=</font> options.lambda_obj <font color='#5555FF'>*</font> out_data[o_idx];
                        <b>}</b>
                    <b>}</b>
                <b>}</b>

                <font color='#009900'>// Now find the best anchor box for each truth box
</font>                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> yolo_rect<font color='#5555FF'>&amp;</font> truth_box : <font color='#5555FF'>*</font>truth<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>truth_box.ignore <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#5555FF'>!</font>input_rect.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font><font color='#BB00BB'>center</font><font face='Lucida Console'>(</font>truth_box.rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;
                    <font color='#0000FF'>const</font> dpoint t_center <font color='#5555FF'>=</font> <font color='#BB00BB'>dcenter</font><font face='Lucida Console'>(</font>truth_box<font face='Lucida Console'>)</font>;
                    <font color='#0000FF'><u>double</u></font> best_iou <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                    <font color='#0000FF'><u>size_t</u></font> best_a <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                    <font color='#0000FF'><u>size_t</u></font> best_tag_id <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> item : options.anchors<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> tag_id <font color='#5555FF'>=</font> item.first;
                        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> details <font color='#5555FF'>=</font> item.second;
                        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> a <font color='#5555FF'>=</font> <font color='#979000'>0</font>; a <font color='#5555FF'>&lt;</font> details.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>a<font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'>const</font> yolo_rect <font color='#BB00BB'>anchor</font><font face='Lucida Console'>(</font><font color='#BB00BB'>centered_drect</font><font face='Lucida Console'>(</font>t_center, details[a].width, details[a].height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> iou <font color='#5555FF'>=</font> <font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>truth_box.rect, anchor.rect<font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>iou <font color='#5555FF'>&gt;</font> best_iou<font face='Lucida Console'>)</font>
                            <b>{</b>
                                best_iou <font color='#5555FF'>=</font> iou;
                                best_a <font color='#5555FF'>=</font> a;
                                best_tag_id <font color='#5555FF'>=</font> tag_id;
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>

                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> a <font color='#5555FF'>=</font> <font color='#979000'>0</font>; a <font color='#5555FF'>&lt;</font> anchors.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>a<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#009900'>// Update best anchor if it's from the current stride, and optionally other anchors
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>best_tag_id <font color='#5555FF'>=</font><font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> best_a <font color='#5555FF'>=</font><font color='#5555FF'>=</font> a<font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> options.iou_anchor_threshold <font color='#5555FF'>&lt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
                        <b>{</b>

                            <font color='#009900'>// do not update other anchors if they have low IoU
</font>                            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font>best_tag_id <font color='#5555FF'>=</font><font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> best_a <font color='#5555FF'>=</font><font color='#5555FF'>=</font> a<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>const</font> yolo_rect <font color='#BB00BB'>anchor</font><font face='Lucida Console'>(</font><font color='#BB00BB'>centered_drect</font><font face='Lucida Console'>(</font>t_center, anchors[a].width, anchors[a].height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> iou <font color='#5555FF'>=</font> <font color='#BB00BB'>box_intersection_over_union</font><font face='Lucida Console'>(</font>truth_box.rect, anchor.rect<font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>iou <font color='#5555FF'>&lt;</font> options.iou_anchor_threshold<font face='Lucida Console'>)</font>
                                    <font color='#0000FF'>continue</font>;
                            <b>}</b>

                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> t_center.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> stride_x;
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> t_center.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> stride_y;
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> a <font color='#5555FF'>*</font> num_feats;

                            <font color='#009900'>// Get the truth box target values
</font>                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> tx <font color='#5555FF'>=</font> t_center.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> stride_x <font color='#5555FF'>-</font> c;
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> ty <font color='#5555FF'>=</font> t_center.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> stride_y <font color='#5555FF'>-</font> r;
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> tw <font color='#5555FF'>=</font> truth_box.rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font>anchors[a].width <font color='#5555FF'>+</font> truth_box.rect.<font color='#BB00BB'>width</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> th <font color='#5555FF'>=</font> truth_box.rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font>anchors[a].height <font color='#5555FF'>+</font> truth_box.rect.<font color='#BB00BB'>height</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                            <font color='#009900'>// Scale regression error according to the truth size
</font>                            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale_box <font color='#5555FF'>=</font> options.lambda_box <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> truth_box.rect.<font color='#BB00BB'>area</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> input_rect.<font color='#BB00BB'>area</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                            <font color='#009900'>// Compute the gradient for the box coordinates
</font>                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> x_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>0</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> y_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>1</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> w_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>2</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> h_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font>;
                            g[x_idx] <font color='#5555FF'>=</font> scale_box <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>out_data[x_idx] <font color='#5555FF'>*</font> <font color='#979000'>2.0</font> <font color='#5555FF'>-</font> <font color='#979000'>0.5</font> <font color='#5555FF'>-</font> tx<font face='Lucida Console'>)</font>;
                            g[y_idx] <font color='#5555FF'>=</font> scale_box <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>out_data[y_idx] <font color='#5555FF'>*</font> <font color='#979000'>2.0</font> <font color='#5555FF'>-</font> <font color='#979000'>0.5</font> <font color='#5555FF'>-</font> ty<font face='Lucida Console'>)</font>;
                            g[w_idx] <font color='#5555FF'>=</font> scale_box <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>out_data[w_idx] <font color='#5555FF'>-</font> tw<font face='Lucida Console'>)</font>;
                            g[h_idx] <font color='#5555FF'>=</font> scale_box <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>out_data[h_idx] <font color='#5555FF'>-</font> th<font face='Lucida Console'>)</font>;

                            <font color='#009900'>// This grid cell should detect an object
</font>                            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> o_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>4</font>, r, c<font face='Lucida Console'>)</font>;
                            g[o_idx] <font color='#5555FF'>=</font> options.lambda_obj <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>out_data[o_idx] <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

                            <font color='#009900'>// Compute the classification error
</font>                            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                            <b>{</b>
                                <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> c_idx <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>output_tensor, n, k <font color='#5555FF'>+</font> <font color='#979000'>5</font> <font color='#5555FF'>+</font> i, r, c<font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>truth_box.label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> options.labels[i]<font face='Lucida Console'>)</font>
                                    g[c_idx] <font color='#5555FF'>=</font> options.lambda_cls <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>out_data[c_idx] <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                                <font color='#0000FF'>else</font>
                                    g[c_idx] <font color='#5555FF'>=</font> options.lambda_cls <font color='#5555FF'>*</font> out_data[c_idx];
                            <b>}</b>
                        <b>}</b>
                    <b>}</b>
                <b>}</b>

                <font color='#009900'>// Compute the L2 loss
</font>                loss <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>length_squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rowm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad<font face='Lucida Console'>)</font>, n<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
    <b>}</b>

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font>... <b><a name='TAG_TYPES'></a>TAG_TYPES</b><font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='loss_yolo_'></a>loss_yolo_</b>
    <b>{</b>
        <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font> <b>{</b> impl::yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>; <b>}</b>

    <font color='#0000FF'>public</font>:

        <font color='#0000FF'>typedef</font> std::vector<font color='#5555FF'>&lt;</font>yolo_rect<font color='#5555FF'>&gt;</font> training_label_type;
        <font color='#0000FF'>typedef</font> std::vector<font color='#5555FF'>&lt;</font>yolo_rect<font color='#5555FF'>&gt;</font> output_label_type;

        constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> impl::yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <b>}</b>

        <b><a name='loss_yolo_'></a>loss_yolo_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>;

        <b><a name='loss_yolo_'></a>loss_yolo_</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> yolo_options<font color='#5555FF'>&amp;</font> options<font face='Lucida Console'>)</font> : options<font face='Lucida Console'>(</font>options<font face='Lucida Console'>)</font> <b>{</b> <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUB_TYPE,
            <font color='#0000FF'>typename</font> label_iterator
            <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='to_label'></a>to_label</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            <font color='#0000FF'>const</font> SUB_TYPE<font color='#5555FF'>&amp;</font> sub,
            label_iterator iter,
            <font color='#0000FF'><u>double</u></font> adjust_threshold <font color='#5555FF'>=</font> <font color='#979000'>0.25</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            std::vector<font color='#5555FF'>&lt;</font>yolo_rect<font color='#5555FF'>&gt;</font> dets_accum;
            std::vector<font color='#5555FF'>&lt;</font>yolo_rect<font color='#5555FF'>&gt;</font> final_dets;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                dets_accum.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                impl::yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tensor_to_dets</font><font face='Lucida Console'>(</font>input_tensor, sub, i, options, adjust_threshold, dets_accum<font face='Lucida Console'>)</font>;

                <font color='#009900'>// Do non-max suppression
</font>                std::<font color='#BB00BB'>sort</font><font face='Lucida Console'>(</font>dets_accum.<font color='#BB00BB'>rbegin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dets_accum.<font color='#BB00BB'>rend</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                final_dets.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> dets_accum.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>overlaps_any_box_nms</font><font face='Lucida Console'>(</font>final_dets, dets_accum[j]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <font color='#0000FF'>continue</font>;

                    final_dets.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>dets_accum[j]<font face='Lucida Console'>)</font>;
                <b>}</b>

                <font color='#5555FF'>*</font>iter<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>final_dets<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> const_label_iterator,
            <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            const_label_iterator truth,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> loss <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                impl::yolo_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tensor_to_loss</font><font face='Lucida Console'>(</font>input_tensor, truth, sub, i, options, loss<font face='Lucida Console'>)</font>;
                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>truth;
            <b>}</b>
            <font color='#0000FF'>return</font> loss <font color='#5555FF'>/</font> input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> yolo_options<font color='#5555FF'>&amp;</font> <b><a name='get_options'></a>get_options</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> options; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='adjust_nms'></a>adjust_nms</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> iou_thresh, <font color='#0000FF'><u>double</u></font> percent_covered_thresh <font color='#5555FF'>=</font> <font color='#979000'>1</font>, <font color='#0000FF'><u>bool</u></font> classwise <font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>
        <b>{</b>
            options.overlaps_nms <font color='#5555FF'>=</font> <font color='#BB00BB'>test_box_overlap</font><font face='Lucida Console'>(</font>iou_thresh, percent_covered_thresh<font face='Lucida Console'>)</font>;
            options.classwise_nms <font color='#5555FF'>=</font> classwise;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_yolo_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_yolo_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>count, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.options, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_yolo_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_yolo_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_yolo_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>count, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>count <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Invalid number of detection tags </font>" <font color='#5555FF'>+</font> std::<font color='#BB00BB'>to_string</font><font face='Lucida Console'>(</font>count<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font>
                                          "<font color='#CC0000'>, while deserializing dlib::loss_yolo_, expecting </font>" <font color='#5555FF'>+</font>
                                          std::<font color='#BB00BB'>to_string</font><font face='Lucida Console'>(</font><font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> "<font color='#CC0000'> tags instead.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.options, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_yolo_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_yolo\t (</font>";
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> opts <font color='#5555FF'>=</font> item.options;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> output</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> ? "<font color='#CC0000'>s</font>" : "<font color='#CC0000'></font>"<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>:(</font>";
            <font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, anchor_boxes:(</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.anchors <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> label</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font>opts.labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> ? "<font color='#CC0000'>s</font>" : "<font color='#CC0000'></font>"<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>:(</font>";
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> opts.labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.labels[i];
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i <font color='#5555FF'>+</font> <font color='#979000'>1</font> <font color='#5555FF'>&lt;</font> opts.labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>,</font>";
            <b>}</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, iou_ignore_threshold: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.iou_ignore_threshold;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, iou_anchor_threshold: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.iou_anchor_threshold;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, lambda_obj:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.lambda_obj;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, lambda_box:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.lambda_box;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, lambda_cls:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.lambda_cls;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, overlaps_nms:(</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.overlaps_nms.<font color='#BB00BB'>get_iou_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>,</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.overlaps_nms.<font color='#BB00BB'>get_percent_covered_thresh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, classwise_nms:</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::boolalpha <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> opts.classwise_nms;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_yolo_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_yolo/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        yolo_options options;

        <font color='#0000FF'>inline</font> <font color='#0000FF'><u>bool</u></font> <b><a name='overlaps_any_box_nms'></a>overlaps_any_box_nms</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>yolo_rect<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> boxes,
            <font color='#0000FF'>const</font> yolo_rect<font color='#5555FF'>&amp;</font> box
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b : boxes<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.<font color='#BB00BB'>overlaps_nms</font><font face='Lucida Console'>(</font>b.rect, box.rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>options.classwise_nms<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>b.label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> box.label<font face='Lucida Console'>)</font>
                            <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
                    <b>}</b>
                    <font color='#0000FF'>else</font>
                    <b>{</b>
                        <font color='#0000FF'>return</font> <font color='#979000'>true</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> <font color='#979000'>false</font>;
        <b>}</b>
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_1'></a>TAG_1</b>, <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_2'></a>TAG_2</b>, <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_3'></a>TAG_3</b>, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_yolo <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_yolo_<font color='#5555FF'>&lt;</font>TAG_1, TAG_2, TAG_3<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='loss_barlow_twins_'></a>loss_barlow_twins_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <b><a name='loss_barlow_twins_'></a>loss_barlow_twins_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#0000FF'>default</font>;

        <b><a name='loss_barlow_twins_'></a>loss_barlow_twins_</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> lambda<font face='Lucida Console'>)</font> : lambda<font face='Lucida Console'>(</font>lambda<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>lambda <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>double</u></font> <b><a name='compute_loss_value_and_gradient'></a>compute_loss_value_and_gradient</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input_tensor,
            SUBNET<font color='#5555FF'>&amp;</font> sub
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tensor<font color='#5555FF'>&amp;</font> grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> sub.<font color='#BB00BB'>sample_expansion_factor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>input_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> batch_size <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#979000'>2</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sample_size <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> offset <font color='#5555FF'>=</font> batch_size <font color='#5555FF'>*</font> sample_size;

            <font color='#009900'>// Alias helpers to access the samples in the batch
</font>            alias_tensor <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>batch_size, sample_size<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> za <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> zb <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>output_tensor, offset<font face='Lucida Console'>)</font>;

            <font color='#009900'>// Normalize both batches independently across the batch dimension
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font>;
            g.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, sample_size<font face='Lucida Console'>)</font>;
            g <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            b.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, sample_size<font face='Lucida Console'>)</font>;
            b <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            tt::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>eps, za_norm, means_a, invstds_a, <font color='#979000'>1</font>, rms, rvs, za, g, b<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>eps, zb_norm, means_b, invstds_b, <font color='#979000'>1</font>, rms, rvs, zb, g, b<font face='Lucida Console'>)</font>;

            <font color='#009900'>// Compute the empirical cross-correlation matrix
</font>            eccm.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>sample_size, sample_size<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, eccm, <font color='#979000'>1</font>, za_norm, <font color='#979000'>true</font>, zb_norm, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            eccm <font color='#5555FF'>/</font><font color='#5555FF'>=</font> batch_size;

            <font color='#009900'>// Set sizes and setup auxiliary tensors
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>eccm, identity<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                identity <font color='#5555FF'>=</font> identity_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sample_size<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>eccm, cdiag<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                cdiag.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>eccm<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>eccm, cdiag_1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                cdiag_1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>eccm<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>eccm, off_mask<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                off_mask <font color='#5555FF'>=</font> ones_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sample_size, sample_size<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> identity_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sample_size<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>eccm, off_diag<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                off_diag.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>eccm<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad, grad_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                grad_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>grad<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>g_grad, g<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                g_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>g<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>b_grad, b<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                b_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>b<font face='Lucida Console'>)</font>;

            <font color='#009900'>// Loss gradient, which will be used as the input of the batch normalization gradient
</font>            <font color='#0000FF'>auto</font> grad_input_a <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>grad_input<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> grad_input_b <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>grad_input, offset<font face='Lucida Console'>)</font>;

            <font color='#009900'>// Compute the loss: notation from http://www.matrixcalculus.org/
</font>            <font color='#009900'>// A = za_norm
</font>            <font color='#009900'>// B = zb_norm
</font>            <font color='#009900'>// C = eccm
</font>            <font color='#009900'>// D = off_mask: a mask that keeps only the elements outside the diagonal
</font>
            <font color='#009900'>// A diagonal matrix containing the diagonal of eccm
</font>            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, cdiag, eccm, identity<font face='Lucida Console'>)</font>;
            <font color='#009900'>// The diagonal of eccm minus the identity matrix
</font>            tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>cdiag_1, cdiag, identity, <font color='#979000'>1</font>, <font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// diagonal term: sum((diag(A' * B) - vector(1)).^2)
</font>            <font color='#009900'>// --------------------------------------------
</font>            <font color='#009900'>// 	=&gt; d/dA = 2 * B * diag(diag(A' * B) - vector(1)) = 2 * B * diag(diag(C) - vector(1))
</font>            <font color='#009900'>// 	=&gt; d/dB = 2 * A * diag(diag(A' * B) - vector(1)) = 2 * A * diag(diag(C) - vector(1))
</font>            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, grad_input_a, <font color='#979000'>2</font>, zb_norm, <font color='#979000'>false</font>, cdiag_1, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, grad_input_b, <font color='#979000'>2</font>, za_norm, <font color='#979000'>false</font>, cdiag_1, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// off-diag term: sum(((A'* B) .* D).^2)
</font>            <font color='#009900'>// --------------------------------
</font>            <font color='#009900'>//  =&gt; d/dA = 2 * B * ((B' * A) .* (D .* D)') = 2 * B * (C .* (D .* D)) = 2 * B * (C .* D)
</font>            <font color='#009900'>//  =&gt; d/dB = 2 * A * ((A' * B) .* (D .* D))  = 2 * A * (C .* (D .* D)) = 2 * A * (C .* D)
</font>            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, off_diag, eccm, off_mask<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, grad_input_a, <font color='#979000'>2</font> <font color='#5555FF'>*</font> lambda, zb_norm, <font color='#979000'>false</font>, off_diag, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, grad_input_b, <font color='#979000'>2</font> <font color='#5555FF'>*</font> lambda, za_norm, <font color='#979000'>false</font>, off_diag, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// Compute the batch norm gradients, g and b grads are not used
</font>            <font color='#0000FF'>auto</font> gza <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>grad<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> gzb <font color='#5555FF'>=</font> <font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>grad, offset<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>eps, grad_input_a, means_a, invstds_a, za, g, gza, g_grad, b_grad<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>eps, grad_input_b, means_b, invstds_b, zb, g, gzb, g_grad, b_grad<font face='Lucida Console'>)</font>;

            <font color='#009900'>// Compute the loss: MSE between eccm and the identity matrix.
</font>            <font color='#009900'>// Off-diagonal terms are weighed by lambda.
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> diagonal_loss <font color='#5555FF'>=</font> <font color='#BB00BB'>sum</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>cdiag_1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> off_diag_loss <font color='#5555FF'>=</font> <font color='#BB00BB'>sum</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>off_diag<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> diagonal_loss <font color='#5555FF'>+</font> lambda <font color='#5555FF'>*</font> off_diag_loss;
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_lambda'></a>get_lambda</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> lambda; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_barlow_twins_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>loss_barlow_twins_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.lambda, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>loss_barlow_twins_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>loss_barlow_twins_</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.lambda, in<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version found while deserializing dlib::loss_barlow_twins_.  Instead found </font>" <font color='#5555FF'>+</font> version<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> loss_barlow_twins_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>loss_barlow_twins (lambda=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.lambda <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> loss_barlow_twins_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;loss_barlow_twins lambda='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.lambda <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>float</u></font> lambda <font color='#5555FF'>=</font> <font color='#979000'>0.0051</font>;
        <font color='#0000FF'>mutable</font> resizable_tensor za_norm, means_a, invstds_a;
        <font color='#0000FF'>mutable</font> resizable_tensor zb_norm, means_b, invstds_b;
        <font color='#0000FF'>mutable</font> resizable_tensor rms, rvs, g, b;
        <font color='#0000FF'>mutable</font> resizable_tensor eccm, grad_input, g_grad, b_grad;
        <font color='#0000FF'>mutable</font> resizable_tensor cdiag, cdiag_1, identity, off_mask, off_diag;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> loss_barlow_twins <font color='#5555FF'>=</font> add_loss_layer<font color='#5555FF'>&lt;</font>loss_barlow_twins_, SUBNET<font color='#5555FF'>&gt;</font>;

<b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_DNn_LOSS_H_
</font>

</pre></body></html>